From 4f8cabf69e0c5c688972613ee8e756d64a7583dc Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 30 Apr 2018 03:13:45 -0500 Subject: [PATCH] move import to local dedicated server --- README.md | 66 ++++++++++++++++++------------------- app/main.go | 26 ++++++++++----- app/rest/api/admin.go | 29 ++++------------ app/rest/api/admin_test.go | 29 ---------------- app/rest/api/import.go | 59 +++++++++++++++++++++++++++++++++ app/rest/api/import_test.go | 63 +++++++++++++++++++++++++++++++++++ app/rest/api/rest.go | 14 ++++---- app/rest/api/rest_test.go | 15 +++++++-- import-disqus.sh | 4 +++ 9 files changed, 200 insertions(+), 105 deletions(-) create mode 100644 app/rest/api/import.go create mode 100644 app/rest/api/import_test.go create mode 100644 import-disqus.sh diff --git a/README.md b/README.md index 964e6bc2..259328e3 100644 --- a/README.md +++ b/README.md @@ -23,34 +23,34 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * copy provided `docker-compose.yml` and customize for your needs * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments -* prepare user id for container ``export USER=`id -u $USER` `` +* prepare user id for container `` export USER=`id -u $USER` `` * make sure you **don't keep** `DEV=true` for any non-development deployments * pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d` * alternatively compile from sources - `docker-compose build` #### Parameters -| Command line | Environment | Default | Multi | Scope | Description | -| --------------- | -------------------- | ---------------------- | ----- | ------ | ------------------------------- | -| --url | REMARK_URL | `https://remark42.com` | no | all | url to remark server | -| --bolt | BOLTDB_PATH | `/tmp` | no | all | path to data directory | -| --site | SITE | `remark` | yes | server | site name(s) | -| --admin | ADMIN | | yes | server | admin names (list of user ids) | -| --backup | BACKUP_PATH | `/tmp` | no | server | backups location | -| --max-back | MAX_BACKUP_FILES | `10` | no | server | max backup files to keep | -| --session | SESSION_STORE | `/tmp` | no | server | path to session store directory | -| --store-key | STORE_KEY | `secure-store-key` | no | server | session store encryption key | -| --google-cid | REMARK_GOOGLE_CID | | no | server | Google OAuth client ID | -| --google-csec | REMARK_GOOGLE_CSEC | | no | server | Google OAuth client secret | -| --facebook-cid | REMARK_FACEBOOK_CID | | no | server | Facebook OAuth client ID | -| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | server | Facebook OAuth client secret | -| --github-cid | REMARK_GITHUB_CID | | no | server | Github OAuth client ID | -| --github-csec | REMARK_GITHUB_CSEC | | no | server | Github OAuth client secret | -| --provider | | `disqus` | no | import | provider type for import | -| --site | | `remark` | no | import | site ID | -| --file | | `disqus.xml` | no | import | import file | -| --dbg | DEBUG | `false` | no | all | debug mode | -| --dev-password | DEV_PASSWD | | no | all | password for `dev` user | +| Command line | Environment | Default | Multi | Description | +| --------------- | -------------------- | ---------------------- | ----- | ------------------------------- | +| --url | REMARK_URL | `https://remark42.com` | no | url to remark server | +| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory | +| --site | SITE | `remark` | yes | site name(s) | +| --admin | ADMIN | | yes | admin names (list of user ids) | +| --backup | BACKUP_PATH | `/tmp` | no | backups location | +| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep | +| --session | SESSION_STORE | `/tmp` | no | path to session store directory | +| --store-key | STORE_KEY | `secure-store-key` | no | session store encryption key | +| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID | +| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret | +| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID | +| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret | +| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID | +| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret | +| --provider | | `disqus` | no | provider type for import | +| --site | | `remark` | no | site ID | +| --file | | `disqus.xml` | no | import file | +| --dbg | DEBUG | `false` | no | debug mode | +| --dev-password | DEV_PASSWD | | no | password for `dev` user | _all multi parameters separated by `,`_ @@ -101,10 +101,8 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu #### Initial import from Disqus 1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details. -2. Move this file to your remark42 host and unzip, i.e. `gunzip .xml.gz`. -3. Stop remark42 containers if started, i.e. `docker-compose stop` -4. Run import command - `docker-compose run --rm remark /srv/remark import --file=.xml --site=` -5. Start remark42 containers `docker-compose up -d` +2. Move this file to your remark42 host within `.var` and unzip, i.e. `gunzip .xml.gz`. +3. Run import command - `docker-compose run remark /srv/import-disqus.sh .xml ` ### Frontend @@ -112,20 +110,20 @@ Frontend part is building automatically along with backend if you use `docker-co For manual building: -- install [Node.js 8](https://nodejs.org/en/) or higher; -- run `npm install` inside `./web`; -- run `npm run build` there; -- result files will be saved in `./web/public`. +* install [Node.js 8](https://nodejs.org/en/) or higher; +* run `npm install` inside `./web`; +* run `npm run build` there; +* result files will be saved in `./web/public`. For development mode use `npm start` instead of `npm run build`. In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`. URLs for development: -- `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`); -- `localhost:8080/dev.html` — page with embedded script from local folder; -- `localhost:8080/last-comments.html` — page with embedded script for last comments; -- `localhost:8080/counter.html` — page with embedded script for counter with examples. +* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`); +* `localhost:8080/dev.html` — page with embedded script from local folder; +* `localhost:8080/last-comments.html` — page with embedded script for last comments; +* `localhost:8080/counter.html` — page with embedded script for counter with examples. ## API diff --git a/app/main.go b/app/main.go index 06b595df..9dce3b80 100644 --- a/app/main.go +++ b/app/main.go @@ -83,7 +83,18 @@ func main() { return sess }() - nativeMigrator := migrator.Remark{CommentFinder: dataStore} + exporter := migrator.Remark{CommentFinder: dataStore} + cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn) + + activateBackup(&exporter) + + importSrv := api.Import{ + Version: revision, + Cache: cache, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + } + go importSrv.Run(opts.Port + 1) avatarProxy := &auth.AvatarProxy{ StorePath: opts.AvatarStore, @@ -92,14 +103,11 @@ func main() { DefaultAvatar: opts.DefaultAvatar, } - activateBackup(&nativeMigrator) - srv := api.Rest{ - Version: revision, - DataService: dataService, - NativeMigrator: nativeMigrator, - DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, - WebRoot: opts.WebRoot, + Version: revision, + DataService: dataService, + Exporter: &exporter, + WebRoot: opts.WebRoot, Authenticator: auth.Authenticator{ Admins: opts.Admins, SessionStore: sessionStore, @@ -107,7 +115,7 @@ func main() { AvatarProxy: avatarProxy, DevPasswd: opts.DevPasswd, }, - Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn), + Cache: cache, } srv.Run(opts.Port) } diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 691f3f67..b98eaf0a 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -18,11 +18,10 @@ import ( // admin provides router for all requests available for admin users only type admin struct { - dataService store.Service - nativeMigrator migrator.Remark - disqusImporter migrator.Importer - cache rest.LoadingCache - defAvatarURL string + dataService store.Service + exporter migrator.Exporter + cache rest.LoadingCache + defAvatarURL string } func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router { @@ -31,7 +30,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route router.Delete("/comment/{id}", a.deleteCommentCtrl) router.Put("/user/{userid}", a.setBlockCtrl) router.Get("/export", a.exportCtrl) - router.Post("/import", a.importCtrl) + router.Put("/pin/{id}", a.setPinCtrl) router.Get("/blocked", a.blockedUsersCtrl) return router @@ -107,27 +106,11 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { writer = gzip.NewWriter(w) } - if err := a.nativeMigrator.Export(writer, siteID); err != nil { + if err := a.exporter.Export(writer, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") } } -// POST /import?site=site-id&provider=disqus|remark -// imports comments from post body. -func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) { - siteID := r.URL.Query().Get("site") - var importer migrator.Importer = &a.nativeMigrator - if r.URL.Query().Get("provider") == "disqus" { - importer = a.disqusImporter - } - - if err := importer.Import(r.Body, siteID); err != nil { - rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") - return - } - a.cache.Flush() -} - func (a *admin) checkBlocked(siteID string, user store.User) bool { return a.dataService.IsBlocked(siteID, user.ID) } diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go index da41c06a..7f39cbc0 100644 --- a/app/rest/api/admin_test.go +++ b/app/rest/api/admin_test.go @@ -160,32 +160,3 @@ func TestAdmin_Export(t *testing.T) { assert.Equal(t, 2, strings.Count(body, "\"text\"")) t.Logf("%s", body) } - -func TestAdmin_Import(t *testing.T) { - srv, port := prep(t) - assert.NotNil(t, srv) - defer cleanup(srv) - - // add 2 initial comments, will be deleted by import - c1 := store.Comment{Text: "test test #1", - Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}} - c2 := store.Comment{Text: "test test #2", - Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}} - addComment(t, c1, port) - addComment(t, c2, port) - - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} - {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) - - resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native", - port), "application/json", r) - assert.Nil(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port)) - assert.Equal(t, 200, code) - pi := []store.PostInfo{} - err = json.Unmarshal([]byte(body), &pi) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi) -} diff --git a/app/rest/api/import.go b/app/rest/api/import.go new file mode 100644 index 00000000..314c3fa7 --- /dev/null +++ b/app/rest/api/import.go @@ -0,0 +1,59 @@ +package api + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/didip/tollbooth" + "github.com/didip/tollbooth_chi" + "github.com/go-chi/chi" + "github.com/go-chi/chi/middleware" + "github.com/umputun/remark/app/migrator" + "github.com/umputun/remark/app/rest" +) + +// Import rest runs on unexposed port and available for local requests only +type Import struct { + Version string + Cache rest.LoadingCache + NativeImporter migrator.Importer + DisqusImporter migrator.Importer + + httpServer *http.Server +} + +// Run the listener and request's router, activate rest server +// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way +func (s *Import) Run(port int) { + log.Printf("[INFO] activate import server on port %d", port) + + router := chi.NewRouter() + router.Use(middleware.RealIP, Recoverer) + router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) + router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil))) + router.Use(AppInfo("remark42-importer", s.Version), Ping, Logger(LogAll)) + + router.Post("/api/v1/admin/import", s.importCtrl) + + s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} + err := s.httpServer.ListenAndServe() + log.Printf("[WARN] http server terminated, %s", err) +} + +// POST /import?site=site-id&provider=disqus|remark +// imports comments from post body. +func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") + importer := s.NativeImporter + if r.URL.Query().Get("provider") == "disqus" { + importer = s.DisqusImporter + } + + if err := importer.Import(r.Body, siteID); err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") + return + } + s.Cache.Flush() +} diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go new file mode 100644 index 00000000..f4045547 --- /dev/null +++ b/app/rest/api/import_test.go @@ -0,0 +1,63 @@ +package api + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/app/migrator" + "github.com/umputun/remark/app/store" +) + +func TestImport(t *testing.T) { + srv, port := prepImportSrv(t) + assert.NotNil(t, srv) + defer cleanupImportSrv(srv) + + r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) + + resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native", + port), "application/json", r) + assert.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func prepImportSrv(t *testing.T) (srv *Import, port int) { + dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"}) + require.Nil(t, err) + + srv = &Import{ + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + Cache: &mockCache{}, + } + + portSetCh := make(chan bool) + + go func() { + port = rand.Intn(50000) + 1025 + portSetCh <- true + srv.Run(port) + }() + + <-portSetCh + + time.Sleep(100 * time.Millisecond) + return srv, port +} + +func cleanupImportSrv(srv *Import) { + srv.httpServer.Close() + srv.httpServer.Shutdown(context.Background()) + os.Remove(testDb) +} diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 664877e5..67287a98 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -35,8 +35,7 @@ type Rest struct { Cache rest.LoadingCache WebRoot string - NativeMigrator migrator.Remark - DisqusImporter migrator.Importer + Exporter migrator.Exporter httpServer *http.Server amdminService admin @@ -44,18 +43,17 @@ type Rest struct { // Run the lister and request's router, activate rest server func (s *Rest) Run(port int) { - log.Print("[INFO] activate rest server") + log.Printf("[INFO] activate rest server on port %d", port) if len(s.Authenticator.Admins) > 0 { log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins) } s.amdminService = admin{ - dataService: s.DataService, - nativeMigrator: s.NativeMigrator, - disqusImporter: s.DisqusImporter, - cache: s.Cache, - defAvatarURL: s.Authenticator.AvatarProxy.Default(), + dataService: s.DataService, + exporter: s.Exporter, + cache: s.Cache, + defAvatarURL: s.Authenticator.AvatarProxy.Default(), } router := chi.NewRouter() diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index e7e34353..714b6c9f 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -421,21 +421,32 @@ func prep(t *testing.T) (srv *Rest, port int) { AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"}, Admins: []string{"a1", "a2"}, }, - NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore}, + Exporter: &migrator.Remark{CommentFinder: dataStore}, + Cache: &mockCache{}, + WebRoot: "/tmp", + } + + importSrv := &Import{ DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, Cache: &mockCache{}, - WebRoot: "/tmp", } ioutil.WriteFile(testHTML, []byte("some html"), 0700) portSetCh := make(chan bool) + go func() { port = rand.Intn(50000) + 1025 portSetCh <- true srv.Run(port) }() + <-portSetCh + go func() { + importSrv.Run(port + 1) + }() + time.Sleep(100 * time.Millisecond) return srv, port } diff --git a/import-disqus.sh b/import-disqus.sh new file mode 100644 index 00000000..cba716dc --- /dev/null +++ b/import-disqus.sh @@ -0,0 +1,4 @@ +#!/bin/sh +echo "import disqus file $1 to site $2" +curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark & +provider=disqus