simplify cmds

This commit is contained in:
Umputun
2018-08-30 00:23:36 -05:00
parent dfa80d46e1
commit 8b3821e797
6 changed files with 150 additions and 138 deletions
+4 -9
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
@@ -37,15 +36,15 @@ func (ec *BackupCommand) Execute(args []string) error {
log.Printf("[DEBUG] export file %s", fname)
// prepare http client
// prepare http client and request
client := http.Client{}
ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
defer cancel()
exportURL := fmt.Sprintf("%s/api/v1/admin/export?site=%s&secret=%s", ec.URL, ec.Site, ec.SharedSecret)
req, err := http.NewRequest(http.MethodGet, exportURL, nil)
if err != nil {
return errors.Wrapf(err, "can't make export request for %s", exportURL)
}
ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
defer cancel()
// get with timeout
resp, err := client.Do(req.WithContext(ctx))
@@ -59,11 +58,7 @@ func (ec *BackupCommand) Execute(args []string) error {
}()
if resp.StatusCode >= 300 {
body, e := ioutil.ReadAll(resp.Body)
if e != nil {
body = []byte("")
}
return errors.Errorf("error response %q, %s", resp.Status, body)
return responseError(resp)
}
fh, err := os.Create(fname)
+11
View File
@@ -4,7 +4,9 @@ package cmd
import (
"bytes"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
@@ -64,3 +66,12 @@ func resetEnv(envs ...string) {
}
}
}
// responseError returns error with status and response body
func responseError(resp *http.Response) error {
body, e := ioutil.ReadAll(resp.Body)
if e != nil {
body = []byte("")
}
return errors.Errorf("error response %q, %s", resp.Status, body)
}
+4 -5
View File
@@ -37,16 +37,15 @@ func (ic *ImportCommand) Execute(args []string) error {
}
client := http.Client{}
ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
defer cancel()
importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s", ic.URL, ic.Site, ic.Provider, ic.SharedSecret)
req, err := http.NewRequest(http.MethodPost, importURL, reader)
if err != nil {
return errors.Wrapf(err, "can't make import request for %s", importURL)
}
ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
defer cancel()
resp, err := client.Do(req.WithContext(ctx)) // closes reader
resp, err := client.Do(req.WithContext(ctx)) // closes request's reader
if err != nil {
return errors.Wrapf(err, "request failed for %s", importURL)
}
@@ -56,7 +55,7 @@ func (ic *ImportCommand) Execute(args []string) error {
}
}()
if resp.StatusCode >= 300 {
return errors.Errorf("error response %s (%d)", resp.Status, resp.StatusCode)
return responseError(resp)
}
body, err := ioutil.ReadAll(resp.Body)
+113 -106
View File
@@ -27,8 +27,8 @@ import (
"github.com/umputun/remark/backend/app/store/service"
)
// ServerOpts with command line flags and env
type ServerOpts struct {
// ServerCommand with command line flags and env
type ServerCommand struct {
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
@@ -117,7 +117,7 @@ var Revision = "unknown"
// serverApp holds all active objects
type serverApp struct {
*ServerOpts
*ServerCommand
restSrv *api.Rest
migratorSrv *api.Migrator
exporter migrator.Exporter
@@ -127,7 +127,7 @@ type serverApp struct {
}
// Execute is the entry point for "server" command, called by flag parser
func (s *ServerOpts) Execute(args []string) error {
func (s *ServerCommand) Execute(args []string) error {
log.Print("[INFO] start remark42 server")
resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
@@ -140,7 +140,7 @@ func (s *ServerOpts) Execute(args []string) error {
cancel()
}()
app, err := newServerApp(s)
app, err := s.newServerApp()
if err != nil {
log.Fatalf("[ERROR] failed to setup application, %+v", err)
}
@@ -154,50 +154,50 @@ func (s *ServerOpts) Execute(args []string) error {
// newServerApp prepares application and return it with all active parts
// doesn't start anything
func newServerApp(opts *ServerOpts) (*serverApp, error) {
func (s *ServerCommand) newServerApp() (*serverApp, error) {
if err := makeDirs(opts.BackupLocation); err != nil {
if err := s.makeDirs(s.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL)
}
storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites)
storeEngine, err := s.makeDataStore()
if err != nil {
return nil, err
}
keyStore, err := makeKeyStore(opts.Key, opts.SharedSecret)
keyStore, err := s.makeKeyStore()
if err != nil {
return nil, err
}
dataService := &service.DataStore{
Interface: storeEngine,
EditDuration: opts.EditDuration,
EditDuration: s.EditDuration,
KeyStore: keyStore,
MaxCommentSize: opts.MaxCommentSize,
Admins: opts.Admins,
MaxCommentSize: s.MaxCommentSize,
Admins: s.Admins,
}
loadingCache, err := makeCache(opts.Cache, opts.Mongo)
loadingCache, err := s.makeCache()
if err != nil {
return nil, err
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(keyStore, strings.HasPrefix(opts.RemarkURL, "https://"), opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie)
jwtService := auth.NewJWT(keyStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie)
avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo)
avatarStore, err := s.makeAvatarStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
RemarkURL: strings.TrimSuffix(s.RemarkURL, "/"),
}
exporter := &migrator.Remark{DataStore: dataService}
@@ -212,26 +212,26 @@ func newServerApp(opts *ServerOpts) (*serverApp, error) {
KeyStore: keyStore,
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
imgProxy := &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL}
authProviders := s.makeAuthProviders(jwtService, avatarProxy, dataService)
imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
commentFormatter := store.NewCommentFormatter(imgProxy)
srv := &api.Rest{
Version: Revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
WebRoot: s.WebRoot,
RemarkURL: s.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
SharedSecret: opts.SharedSecret,
ReadOnlyAge: s.ReadOnlyAge,
SharedSecret: s.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
AdminEmail: s.AdminEmail,
Providers: authProviders,
DevPasswd: opts.DevPasswd,
DevPasswd: s.DevPasswd,
PermissionChecker: dataService,
},
Cache: loadingCache,
@@ -239,25 +239,32 @@ func newServerApp(opts *ServerOpts) (*serverApp, error) {
// no admin email, use admin@domain
if srv.Authenticator.AdminEmail == "" {
if u, err := url.Parse(opts.RemarkURL); err == nil {
if u, err := url.Parse(s.RemarkURL); err == nil {
srv.Authenticator.AdminEmail = "admin@" + u.Host
}
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
var devAuth *auth.DevAuthServer
if opts.Auth.Dev {
if s.Auth.Dev {
devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]}
}
tch := make(chan struct{})
return &serverApp{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService,
ServerOpts: opts, terminated: tch}, nil
return &serverApp{
ServerCommand: s,
restSrv: srv,
migratorSrv: migr,
exporter: exporter,
devAuth: devAuth,
dataService: dataService,
terminated: make(chan struct{}),
}, nil
}
// Run all application objects
func (a *serverApp) run(ctx context.Context) error {
log.Printf("%+v", a)
if a.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
@@ -305,76 +312,119 @@ func (a *serverApp) activateBackup(ctx context.Context) {
}
// makeDataStore creates store for all sites
func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) {
switch group.Type {
func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
switch s.Store.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
if err = s.makeDirs(s.Store.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
for _, site := range s.Sites {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
case "mongo":
mgServer, e := makeMongo(mg)
mgServer, e := s.makeMongo()
if e != nil {
return result, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
}
return result, errors.Wrap(err, "can't initialize data store")
}
func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) {
switch group.Type {
func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
switch s.Avatar.Type {
case "fs":
if err := makeDirs(group.FS.Path); err != nil {
if err := s.makeDirs(s.Avatar.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil
return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil
case "mongo":
mgServer, err := makeMongo(mg)
mgServer, err := s.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "")
return avatar.NewGridFS(conn, group.RszLmt), nil
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
return avatar.NewGridFS(conn, s.Avatar.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
}
func makeKeyStore(group KeyGroup, sharedSecret string) (keys.Store, error) {
switch group.Type {
func (s *ServerCommand) makeKeyStore() (keys.Store, error) {
switch s.Key.Type {
case "shared":
return keys.NewStaticStore(sharedSecret), nil
return keys.NewStaticStore(s.SharedSecret), nil
default:
return nil, errors.Errorf("unsupported key store type %s", group.Type)
return nil, errors.Errorf("unsupported key store type %s", s.Key.Type)
}
}
func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) {
switch group.Type {
func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
switch s.Cache.Type {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
case "mongo":
mgServer, err := makeMongo(mg)
mgServer, err := s.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, mg.DB, "cache")
return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
cache.MaxKeys(group.Max.Items))
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
}
return nil, errors.Errorf("unsupported cache type %s", group.Type)
return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
}
func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
if s.Mongo.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
}
func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
RemarkURL: s.RemarkURL,
Cid: cid,
Csecret: secret,
PermissionChecker: ds,
}
}
providers := []auth.Provider{}
if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(s.Auth.Google.CID, s.Auth.Google.CSEC)))
}
if s.Auth.Github.CID != "" && s.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(s.Auth.Github.CID, s.Auth.Github.CSEC)))
}
if s.Auth.Facebook.CID != "" && s.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(s.Auth.Facebook.CID, s.Auth.Facebook.CSEC)))
}
if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)))
}
if s.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
func (s *ServerCommand) makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
@@ -401,46 +451,3 @@ func makeDirs(dirs ...string) error {
}
return nil
}
func makeMongo(mg MongoGroup) (result *mongo.Server, err error) {
if mg.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(mg.URL, 10*time.Second)
}
func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts *ServerOpts) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
RemarkURL: opts.RemarkURL,
Cid: cid,
Csecret: secret,
PermissionChecker: ds,
}
}
providers := []auth.Provider{}
if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
}
if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
}
if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
}
if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
}
if opts.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
+17 -17
View File
@@ -20,7 +20,7 @@ import (
)
func TestServerApp(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18080
return o
})
@@ -51,7 +51,7 @@ func TestServerApp(t *testing.T) {
}
func TestServerApp_DevMode(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18085
o.DevPasswd = "password"
o.Auth.Dev = true
@@ -85,7 +85,7 @@ func TestServerApp_WithMongo(t *testing.T) {
t.Skip("skip mongo app test")
}
opts := ServerOpts{}
opts := ServerCommand{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com",
@@ -95,7 +95,7 @@ func TestServerApp_WithMongo(t *testing.T) {
opts.BackupLocation = "/tmp"
// create app
app, err := newServerApp(&opts)
app, err := opts.newServerApp()
require.Nil(t, err)
defer func() {
@@ -130,47 +130,47 @@ func TestServerApp_WithMongo(t *testing.T) {
}
func TestServerApp_Failed(t *testing.T) {
opts := ServerOpts{}
opts := ServerCommand{}
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp",
"--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
_, err = opts.newServerApp()
assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = ServerOpts{}
opts = ServerCommand{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
"--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
_, err = opts.newServerApp()
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = ServerOpts{}
opts = ServerCommand{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = newServerApp(&opts)
_, err = opts.newServerApp()
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = ServerOpts{}
opts = ServerCommand{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"})
assert.NotNil(t, err, "blah is invalid type")
opts.Store.Type = "blah"
_, err = newServerApp(&opts)
_, err = opts.newServerApp()
assert.EqualError(t, err, "unsupported store type blah")
t.Log(err)
}
func TestServerApp_Shutdown(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerOpts) ServerOpts {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18090
return o
})
@@ -190,7 +190,7 @@ func TestServerApp_MainSignal(t *testing.T) {
}()
st := time.Now()
s := ServerOpts{}
s := ServerCommand{}
p := flags.NewParser(&s, flags.Default)
args := []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
"--port=18100", "--url=https://demo.remark42.com"}
@@ -201,8 +201,8 @@ func TestServerApp_MainSignal(t *testing.T) {
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
}
func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) ServerOpts) (*serverApp, context.Context) {
opts := ServerOpts{}
func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
opts := ServerCommand{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
@@ -220,7 +220,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerOpts) S
os.Remove(opts.Store.Bolt.Path + "/remark.db")
// create app
app, err := newServerApp(&opts)
app, err := opts.newServerApp()
require.Nil(t, err)
ctx, cancel := context.WithCancel(context.Background())
+1 -1
View File
@@ -13,7 +13,7 @@ import (
// Opts has all commands
type Opts struct {
ServerCmd cmd.ServerOpts `command:"server"`
ServerCmd cmd.ServerCommand `command:"server"`
ImportCmd cmd.ImportCommand `command:"import"`
BackupCmd cmd.BackupCommand `command:"backup"`
RestoreCmd cmd.RestoreCommand `command:"restore"`