From 7934d9af736c98e5c74c83b52e9d29bb64caef89 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Sep 2018 20:48:36 -0500 Subject: [PATCH] add cli for AvatarCmd --- backend/app/cmd/avatar.go | 84 ++++++++++++++++++++++++++++++++++ backend/app/cmd/avatar_test.go | 53 +++++++++++++++++++++ backend/app/cmd/cmd.go | 29 ++++++++++++ backend/app/cmd/server.go | 35 ++------------ backend/app/main.go | 1 + 5 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 backend/app/cmd/avatar.go create mode 100644 backend/app/cmd/avatar_test.go diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go new file mode 100644 index 00000000..c4cd506d --- /dev/null +++ b/backend/app/cmd/avatar.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "log" + "time" + + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store/avatar" +) + +// AvatarCommand set of flags and command for avatar migration +// it converts all avatarts from src.type to dst.type +type AvatarCommand struct { + AvatarSrc AvatarGroup `group:"src" namespace:"src"` + AvatarDst AvatarGroup `group:"dst" namespace:"dst"` + Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + + migrator AvatarMigrator + CommonOpts +} + +// AvatarMigrator defines interface for migration +type AvatarMigrator interface { + Migrate(avatar.Store, avatar.Store) (int, error) +} + +type avatarMigrator struct{} + +func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) { + return avatar.Migrate(dst, src) +} + +// Execute runs with AvatarCommand parameters, entry point for "avatar" command +func (ac *AvatarCommand) Execute(args []string) error { + log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type) + + src, err := ac.makeAvatarStore(ac.AvatarSrc) + if err != nil { + return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type) + } + + dst, err := ac.makeAvatarStore(ac.AvatarDst) + if err != nil { + return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type) + } + + if ac.migrator == nil { + ac.migrator = avatarMigrator{} + } + + count, err := ac.migrator.Migrate(dst, src) + if err != nil { + return err + } + log.Printf("[INFO] completed, migrated avatars = %d", count) + return nil +} + +func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { + switch gr.Type { + case "fs": + if err := makeDirs(gr.FS.Path); err != nil { + return nil, err + } + return avatar.NewLocalFS(gr.FS.Path, gr.RszLmt), nil + case "mongo": + mgServer, err := ac.makeMongo() + if err != nil { + return nil, errors.Wrap(err, "failed to create mongo server") + } + conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "") + return avatar.NewGridFS(conn, gr.RszLmt), nil + } + return nil, errors.Errorf("unsupported avatar store type %s", gr.Type) +} + +func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) { + if ac.Mongo.URL == "" { + return nil, errors.New("no mongo URL provided") + } + return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second) +} diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go new file mode 100644 index 00000000..5362ae08 --- /dev/null +++ b/backend/app/cmd/avatar_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "errors" + "os" + "testing" + + flags "github.com/jessevdk/go-flags" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/avatar" +) + +func TestAvatar_Execute(t *testing.T) { + + mongoURL := os.Getenv("MONGO_TEST") + if mongoURL == "" { + mongoURL = "mongodb://localhost:27017/test" + } + if mongoURL == "skip" { + t.Skip("skip mongo app test") + } + defer os.RemoveAll("/tmp/ava-test") + + cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} + cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) + p := flags.NewParser(&cmd, flags.Default) + _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", + "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.NoError(t, err) + + cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}} + cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) + p = flags.NewParser(&cmd, flags.Default) + _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", + "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) + require.Nil(t, err) + err = cmd.Execute(nil) + assert.Error(t, err, "failed blah") +} + +type avatarMigratorMock struct { + called int + retError error + retCount int +} + +func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) { + a.called++ + return a.retCount, a.retError +} diff --git a/backend/app/cmd/cmd.go b/backend/app/cmd/cmd.go index 664dba0b..0b688828 100644 --- a/backend/app/cmd/cmd.go +++ b/backend/app/cmd/cmd.go @@ -98,3 +98,32 @@ func responseError(resp *http.Response) error { } return errors.Errorf("error response %q, %s", resp.Status, body) } + +// mkdir -p for all dirs +func makeDirs(dirs ...string) error { + + // exists returns whether the given file or directory exists or not + exists := func(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return true, err + } + + for _, dir := range dirs { + ex, err := exists(dir) + if err != nil { + return errors.Wrapf(err, "can't check directory status for %s", dir) + } + if !ex { + if e := os.MkdirAll(dir, 0700); e != nil { + return errors.Wrapf(err, "can't make directory %s", dir) + } + } + } + return nil +} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index b1d7ab2f..acae46b8 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -160,7 +160,7 @@ func (s *ServerCommand) Execute(args []string) error { // doesn't start anything func (s *ServerCommand) newServerApp() (*serverApp, error) { - if err := s.makeDirs(s.BackupLocation); err != nil { + if err := makeDirs(s.BackupLocation); err != nil { return nil, err } @@ -315,7 +315,7 @@ func (a *serverApp) activateBackup(ctx context.Context) { func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { switch s.Store.Type { case "bolt": - if err = s.makeDirs(s.Store.Bolt.Path); err != nil { + if err = makeDirs(s.Store.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } sites := []engine.BoltSite{} @@ -339,7 +339,7 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { switch s.Avatar.Type { case "fs": - if err := s.makeDirs(s.Avatar.FS.Path); err != nil { + if err := makeDirs(s.Avatar.FS.Path); err != nil { return nil, err } return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil @@ -451,32 +451,3 @@ func (s *ServerCommand) makeAuthProviders(jwtService *auth.JWT, avatarProxy *pro } return providers } - -// mkdir -p for all dirs -func (s *ServerCommand) makeDirs(dirs ...string) error { - - // exists returns whether the given file or directory exists or not - exists := func(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err - } - - for _, dir := range dirs { - ex, err := exists(dir) - if err != nil { - return errors.Wrapf(err, "can't check directory status for %s", dir) - } - if !ex { - if e := os.MkdirAll(dir, 0700); e != nil { - return errors.Wrapf(err, "can't make directory %s", dir) - } - } - } - return nil -} diff --git a/backend/app/main.go b/backend/app/main.go index 36040c70..33151e55 100644 --- a/backend/app/main.go +++ b/backend/app/main.go @@ -17,6 +17,7 @@ type Opts struct { ImportCmd cmd.ImportCommand `command:"import"` BackupCmd cmd.BackupCommand `command:"backup"` RestoreCmd cmd.RestoreCommand `command:"restore"` + AvatarCmd cmd.AvatarCommand `command:"avatar"` RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"` SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`