From 15095a915150b05c9f11ae024fca3dc4150b0597 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 30 Apr 2018 02:14:14 -0500 Subject: [PATCH] remove import cmd, add import test --- app/main.go | 98 ++++++++++++++------------------------ app/rest/api/admin.go | 12 ++--- app/rest/api/admin_test.go | 29 +++++++++++ app/rest/api/rest.go | 10 ++-- app/rest/api/rest_test.go | 7 +-- remark.rest | 1 - start.sh | 2 +- 7 files changed, 81 insertions(+), 78 deletions(-) diff --git a/app/main.go b/app/main.go index e3a87156..06b595df 100644 --- a/app/main.go +++ b/app/main.go @@ -33,30 +33,22 @@ var opts struct { BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var" description:"backups location"` MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` - ServerCommand struct { - SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"` - StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"` + SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"` + StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"` - GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"` - GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"` - GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"` - GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"` - FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"` - FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"` - DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"` - DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"` + GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"` + GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"` + GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"` + GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"` + FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"` + FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"` + DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"` + DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"` - AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"` - DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"` - Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` - WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` - } `command:"server" description:"run server"` - - ImportCommand struct { - Provider string `long:"provider" default:"disqus" description:"provider type"` - SiteID string `long:"site" default:"remark" description:"site ID"` - InputFile string `long:"file" default:"disqus.xml" description:"input file"` - } `command:"import" description:"import comments from external sources"` + AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"` + DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"` + Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` + WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` } var revision = "unknown" @@ -71,33 +63,19 @@ func main() { setupLog(opts.Dbg) log.Print("[INFO] started remark") - if err := makeDirs(opts.BoltPath, opts.ServerCommand.SessionStore, opts.BackupLocation, opts.ServerCommand.AvatarStore); err != nil { + if err := makeDirs(opts.BoltPath, opts.SessionStore, opts.BackupLocation, opts.AvatarStore); err != nil { log.Fatalf("[ERROR] can't create directories, %+v", err) } dataStore := makeBoltStore(opts.Sites) - if p.Active != nil && p.Command.Find("import") == p.Active { - // import mode - params := migrator.ImportParams{ - CommentCreator: dataStore, - InputFile: opts.ImportCommand.InputFile, - Provider: opts.ImportCommand.Provider, - SiteID: opts.ImportCommand.SiteID, - } - if err := migrator.ImportComments(params); err != nil { - log.Fatalf("[ERROR] failed to import, %+v", err) - } - return - } - if opts.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } - dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.ServerCommand.StoreKey} + dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.StoreKey} sessionStore := func() sessions.Store { - sess := sessions.NewFilesystemStore(opts.ServerCommand.SessionStore, []byte(opts.ServerCommand.StoreKey)) + sess := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey)) sess.Options.HttpOnly = true sess.Options.Secure = true sess.Options.MaxAge = 3600 * 24 * 365 @@ -105,24 +83,23 @@ func main() { return sess }() - exporter := &migrator.Remark{CommentFinder: dataStore} + nativeMigrator := migrator.Remark{CommentFinder: dataStore} avatarProxy := &auth.AvatarProxy{ - StorePath: opts.ServerCommand.AvatarStore, + StorePath: opts.AvatarStore, RoutePath: "/api/v1/avatar", RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), - DefaultAvatar: opts.ServerCommand.DefaultAvatar, + DefaultAvatar: opts.DefaultAvatar, } - activateBackup(exporter) + activateBackup(&nativeMigrator) srv := api.Rest{ - Version: revision, - DataService: dataService, - Exporter: exporter, - Importer: &migrator.Remark{CommentCreator: dataStore}, - Disqus: &migrator.Disqus{CommentCreator: dataStore}, - WebRoot: opts.ServerCommand.WebRoot, + Version: revision, + DataService: dataService, + NativeMigrator: nativeMigrator, + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + WebRoot: opts.WebRoot, Authenticator: auth.Authenticator{ Admins: opts.Admins, SessionStore: sessionStore, @@ -132,7 +109,7 @@ func main() { }, Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn), } - srv.Run(opts.ServerCommand.Port) + srv.Run(opts.Port) } // activateBackup runs background backups for each site @@ -203,19 +180,17 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx } } - srvOpts := opts.ServerCommand - - if srvOpts.GoogleCID != "" && srvOpts.GoogleCSEC != "" { - providers = append(providers, auth.NewGoogle(makeParams(srvOpts.GoogleCID, srvOpts.GoogleCSEC))) + if opts.GoogleCID != "" && opts.GoogleCSEC != "" { + providers = append(providers, auth.NewGoogle(makeParams(opts.GoogleCID, opts.GoogleCSEC))) } - if srvOpts.GithubCID != "" && srvOpts.GithubCSEC != "" { - providers = append(providers, auth.NewGithub(makeParams(srvOpts.GithubCID, srvOpts.GithubCSEC))) + if opts.GithubCID != "" && opts.GithubCSEC != "" { + providers = append(providers, auth.NewGithub(makeParams(opts.GithubCID, opts.GithubCSEC))) } - if srvOpts.FacebookCID != "" && srvOpts.FacebookCSEC != "" { - providers = append(providers, auth.NewFacebook(makeParams(srvOpts.FacebookCID, srvOpts.FacebookCSEC))) + if opts.FacebookCID != "" && opts.FacebookCSEC != "" { + providers = append(providers, auth.NewFacebook(makeParams(opts.FacebookCID, opts.FacebookCSEC))) } - if srvOpts.DisqusCID != "" && srvOpts.DisqusCSEC != "" { - providers = append(providers, auth.NewDisqus(makeParams(srvOpts.DisqusCID, srvOpts.DisqusCSEC))) + if opts.DisqusCID != "" && opts.DisqusCSEC != "" { + providers = append(providers, auth.NewDisqus(makeParams(opts.DisqusCID, opts.DisqusCSEC))) } if len(providers) == 0 { log.Printf("[WARN] no auth providers defined") @@ -226,6 +201,7 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx // post-flush callback invoked by cache after each flush in async way func postFlushFn() { + // list of heavy urls for pre-heating on cache change urls := []string{ "http://localhost:%d/api/v1/list?site=%s", "http://localhost:%d/api/v1/last/50?site=%s", @@ -233,7 +209,7 @@ func postFlushFn() { for _, site := range opts.Sites { for _, u := range urls { - resp, err := http.Get(fmt.Sprintf(u, opts.ServerCommand.Port, site)) + resp, err := http.Get(fmt.Sprintf(u, opts.Port, site)) if err != nil { log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err) return diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 8eada1e8..691f3f67 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -19,9 +19,8 @@ import ( // admin provides router for all requests available for admin users only type admin struct { dataService store.Service - exporterNative migrator.Exporter - importerNative migrator.Importer - importerDisqus migrator.Importer + nativeMigrator migrator.Remark + disqusImporter migrator.Importer cache rest.LoadingCache defAvatarURL string } @@ -108,7 +107,7 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { writer = gzip.NewWriter(w) } - if err := a.exporterNative.Export(writer, siteID); err != nil { + if err := a.nativeMigrator.Export(writer, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") } } @@ -117,13 +116,14 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { // imports comments from post body. func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") - importer := a.importerNative + var importer migrator.Importer = &a.nativeMigrator if r.URL.Query().Get("provider") == "disqus" { - importer = a.importerDisqus + importer = a.disqusImporter } if err := importer.Import(r.Body, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") + return } a.cache.Flush() } diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go index 7f39cbc0..da41c06a 100644 --- a/app/rest/api/admin_test.go +++ b/app/rest/api/admin_test.go @@ -160,3 +160,32 @@ func TestAdmin_Export(t *testing.T) { assert.Equal(t, 2, strings.Count(body, "\"text\"")) t.Logf("%s", body) } + +func TestAdmin_Import(t *testing.T) { + srv, port := prep(t) + assert.NotNil(t, srv) + defer cleanup(srv) + + // add 2 initial comments, will be deleted by import + c1 := store.Comment{Text: "test test #1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}} + c2 := store.Comment{Text: "test test #2", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}} + addComment(t, c1, port) + addComment(t, c2, port) + + r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) + + resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native", + port), "application/json", r) + assert.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port)) + assert.Equal(t, 200, code) + pi := []store.PostInfo{} + err = json.Unmarshal([]byte(body), &pi) + assert.Nil(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi) +} diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index bc7822d3..664877e5 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -35,9 +35,8 @@ type Rest struct { Cache rest.LoadingCache WebRoot string - Exporter migrator.Exporter - Importer migrator.Importer - Disqus migrator.Importer + NativeMigrator migrator.Remark + DisqusImporter migrator.Importer httpServer *http.Server amdminService admin @@ -53,9 +52,8 @@ func (s *Rest) Run(port int) { s.amdminService = admin{ dataService: s.DataService, - exporterNative: s.Exporter, - importerNative: s.Importer, - importerDisqus: s.Disqus, + nativeMigrator: s.NativeMigrator, + disqusImporter: s.DisqusImporter, cache: s.Cache, defAvatarURL: s.Authenticator.AvatarProxy.Default(), } diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index fa110ddb..e7e34353 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -421,9 +421,10 @@ func prep(t *testing.T) (srv *Rest, port int) { AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"}, Admins: []string{"a1", "a2"}, }, - Exporter: &migrator.Remark{CommentFinder: dataStore}, - Cache: &mockCache{}, - WebRoot: "/tmp", + NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore}, + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + Cache: &mockCache{}, + WebRoot: "/tmp", } ioutil.WriteFile(testHTML, []byte("some html"), 0700) diff --git a/remark.rest b/remark.rest index a52a2126..206679a5 100644 --- a/remark.rest +++ b/remark.rest @@ -107,4 +107,3 @@ GET {{host}}/api/v1/avatar/blah ### ping GET {{host}}/ping - diff --git a/start.sh b/start.sh index b8ee8fa0..3b87cb73 100755 --- a/start.sh +++ b/start.sh @@ -12,4 +12,4 @@ cp -nv /srv/web/remark.image /srv/var/avatars/remark.image echo "start remark42 server" -/sbin/su-exec ${USER} /srv/remark server +/sbin/su-exec ${USER} /srv/remark