diff --git a/README.md b/README.md index 964e6bc2..2e2600f4 100644 --- a/README.md +++ b/README.md @@ -23,42 +23,37 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi * copy provided `docker-compose.yml` and customize for your needs * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments -* prepare user id for container ``export USER=`id -u $USER` `` +* prepare user id for container `` export USER=`id -u $USER` `` * make sure you **don't keep** `DEV=true` for any non-development deployments * pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d` * alternatively compile from sources - `docker-compose build` #### Parameters -| Command line | Environment | Default | Multi | Scope | Description | -| --------------- | -------------------- | ---------------------- | ----- | ------ | ------------------------------- | -| --url | REMARK_URL | `https://remark42.com` | no | all | url to remark server | -| --bolt | BOLTDB_PATH | `/tmp` | no | all | path to data directory | -| --site | SITE | `remark` | yes | server | site name(s) | -| --admin | ADMIN | | yes | server | admin names (list of user ids) | -| --backup | BACKUP_PATH | `/tmp` | no | server | backups location | -| --max-back | MAX_BACKUP_FILES | `10` | no | server | max backup files to keep | -| --session | SESSION_STORE | `/tmp` | no | server | path to session store directory | -| --store-key | STORE_KEY | `secure-store-key` | no | server | session store encryption key | -| --google-cid | REMARK_GOOGLE_CID | | no | server | Google OAuth client ID | -| --google-csec | REMARK_GOOGLE_CSEC | | no | server | Google OAuth client secret | -| --facebook-cid | REMARK_FACEBOOK_CID | | no | server | Facebook OAuth client ID | -| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | server | Facebook OAuth client secret | -| --github-cid | REMARK_GITHUB_CID | | no | server | Github OAuth client ID | -| --github-csec | REMARK_GITHUB_CSEC | | no | server | Github OAuth client secret | -| --provider | | `disqus` | no | import | provider type for import | -| --site | | `remark` | no | import | site ID | -| --file | | `disqus.xml` | no | import | import file | -| --dbg | DEBUG | `false` | no | all | debug mode | -| --dev-password | DEV_PASSWD | | no | all | password for `dev` user | +| Command line | Environment | Default | Multi | Description | +| --------------- | -------------------- | ---------------------- | ----- | ------------------------------- | +| --url | REMARK_URL | `https://remark42.com` | no | url to remark server | +| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory | +| --site | SITE | `remark` | yes | site name(s) | +| --admin | ADMIN | | yes | admin names (list of user ids) | +| --backup | BACKUP_PATH | `/tmp` | no | backups location | +| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep | +| --session | SESSION_STORE | `/tmp` | no | path to session store directory | +| --store-key | STORE_KEY | `secure-store-key` | no | session store encryption key | +| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID | +| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret | +| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID | +| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret | +| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID | +| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret | +| --provider | | `disqus` | no | provider type for import | +| --site | | `remark` | no | site ID | +| --file | | `disqus.xml` | no | import file | +| --dbg | DEBUG | `false` | no | debug mode | +| --dev-password | DEV_PASSWD | | no | password for `dev` user | _all multi parameters separated by `,`_ -#### Run modes - -* `server` activates regular, server mode -* `import` performs import from external providers (disqus and internal json, see `/api/v1/admin/export`) - #### Register oauth2 providers Authentication handled by external providers. You should setup oauth2 for all (or some) of them in order to allow users to make comments. It is not mandatory to have all of them, but at least one should be property configured. @@ -101,10 +96,8 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu #### Initial import from Disqus 1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details. -2. Move this file to your remark42 host and unzip, i.e. `gunzip .xml.gz`. -3. Stop remark42 containers if started, i.e. `docker-compose stop` -4. Run import command - `docker-compose run --rm remark /srv/remark import --file=.xml --site=` -5. Start remark42 containers `docker-compose up -d` +2. Move this file to your remark42 host within `.var` and unzip, i.e. `gunzip .xml.gz`. +3. Run import command - `docker-compose run remark /srv/import-disqus.sh .xml ` ### Frontend @@ -112,20 +105,20 @@ Frontend part is building automatically along with backend if you use `docker-co For manual building: -- install [Node.js 8](https://nodejs.org/en/) or higher; -- run `npm install` inside `./web`; -- run `npm run build` there; -- result files will be saved in `./web/public`. +* install [Node.js 8](https://nodejs.org/en/) or higher; +* run `npm install` inside `./web`; +* run `npm run build` there; +* result files will be saved in `./web/public`. For development mode use `npm start` instead of `npm run build`. In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`. URLs for development: -- `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`); -- `localhost:8080/dev.html` — page with embedded script from local folder; -- `localhost:8080/last-comments.html` — page with embedded script for last comments; -- `localhost:8080/counter.html` — page with embedded script for counter with examples. +* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`); +* `localhost:8080/dev.html` — page with embedded script from local folder; +* `localhost:8080/last-comments.html` — page with embedded script for last comments; +* `localhost:8080/counter.html` — page with embedded script for counter with examples. ## API diff --git a/app/main.go b/app/main.go index 333b7e38..9dce3b80 100644 --- a/app/main.go +++ b/app/main.go @@ -33,30 +33,22 @@ var opts struct { BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var" description:"backups location"` MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"` - ServerCommand struct { - SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"` - StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"` + SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"` + StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"` - GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"` - GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"` - GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"` - GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"` - FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"` - FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"` - DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"` - DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"` + GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"` + GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"` + GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"` + GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"` + FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"` + FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"` + DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"` + DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"` - AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"` - DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"` - Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` - WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` - } `command:"server" description:"run server"` - - ImportCommand struct { - Provider string `long:"provider" default:"disqus" description:"provider type"` - SiteID string `long:"site" default:"remark" description:"site ID"` - InputFile string `long:"file" default:"disqus.xml" description:"input file"` - } `command:"import" description:"import comments from external sources"` + AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"` + DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"` + Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"` + WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"` } var revision = "unknown" @@ -71,33 +63,19 @@ func main() { setupLog(opts.Dbg) log.Print("[INFO] started remark") - if err := makeDirs(opts.BoltPath, opts.ServerCommand.SessionStore, opts.BackupLocation, opts.ServerCommand.AvatarStore); err != nil { + if err := makeDirs(opts.BoltPath, opts.SessionStore, opts.BackupLocation, opts.AvatarStore); err != nil { log.Fatalf("[ERROR] can't create directories, %+v", err) } dataStore := makeBoltStore(opts.Sites) - if p.Active != nil && p.Command.Find("import") == p.Active { - // import mode - params := migrator.ImportParams{ - CommentCreator: dataStore, - InputFile: opts.ImportCommand.InputFile, - Provider: opts.ImportCommand.Provider, - SiteID: opts.ImportCommand.SiteID, - } - if err := migrator.ImportComments(params); err != nil { - log.Fatalf("[ERROR] failed to import, %+v", err) - } - return - } - if opts.DevPasswd != "" { log.Printf("[WARN] running in dev mode") } - dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.ServerCommand.StoreKey} + dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.StoreKey} sessionStore := func() sessions.Store { - sess := sessions.NewFilesystemStore(opts.ServerCommand.SessionStore, []byte(opts.ServerCommand.StoreKey)) + sess := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey)) sess.Options.HttpOnly = true sess.Options.Secure = true sess.Options.MaxAge = 3600 * 24 * 365 @@ -105,22 +83,31 @@ func main() { return sess }() - exporter := &migrator.Remark{CommentFinder: dataStore} + exporter := migrator.Remark{CommentFinder: dataStore} + cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn) + + activateBackup(&exporter) + + importSrv := api.Import{ + Version: revision, + Cache: cache, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + } + go importSrv.Run(opts.Port + 1) avatarProxy := &auth.AvatarProxy{ - StorePath: opts.ServerCommand.AvatarStore, + StorePath: opts.AvatarStore, RoutePath: "/api/v1/avatar", RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"), - DefaultAvatar: opts.ServerCommand.DefaultAvatar, + DefaultAvatar: opts.DefaultAvatar, } - activateBackup(exporter) - srv := api.Rest{ Version: revision, DataService: dataService, - Exporter: exporter, - WebRoot: opts.ServerCommand.WebRoot, + Exporter: &exporter, + WebRoot: opts.WebRoot, Authenticator: auth.Authenticator{ Admins: opts.Admins, SessionStore: sessionStore, @@ -128,9 +115,9 @@ func main() { AvatarProxy: avatarProxy, DevPasswd: opts.DevPasswd, }, - Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn), + Cache: cache, } - srv.Run(opts.ServerCommand.Port) + srv.Run(opts.Port) } // activateBackup runs background backups for each site @@ -201,19 +188,17 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx } } - srvOpts := opts.ServerCommand - - if srvOpts.GoogleCID != "" && srvOpts.GoogleCSEC != "" { - providers = append(providers, auth.NewGoogle(makeParams(srvOpts.GoogleCID, srvOpts.GoogleCSEC))) + if opts.GoogleCID != "" && opts.GoogleCSEC != "" { + providers = append(providers, auth.NewGoogle(makeParams(opts.GoogleCID, opts.GoogleCSEC))) } - if srvOpts.GithubCID != "" && srvOpts.GithubCSEC != "" { - providers = append(providers, auth.NewGithub(makeParams(srvOpts.GithubCID, srvOpts.GithubCSEC))) + if opts.GithubCID != "" && opts.GithubCSEC != "" { + providers = append(providers, auth.NewGithub(makeParams(opts.GithubCID, opts.GithubCSEC))) } - if srvOpts.FacebookCID != "" && srvOpts.FacebookCSEC != "" { - providers = append(providers, auth.NewFacebook(makeParams(srvOpts.FacebookCID, srvOpts.FacebookCSEC))) + if opts.FacebookCID != "" && opts.FacebookCSEC != "" { + providers = append(providers, auth.NewFacebook(makeParams(opts.FacebookCID, opts.FacebookCSEC))) } - if srvOpts.DisqusCID != "" && srvOpts.DisqusCSEC != "" { - providers = append(providers, auth.NewDisqus(makeParams(srvOpts.DisqusCID, srvOpts.DisqusCSEC))) + if opts.DisqusCID != "" && opts.DisqusCSEC != "" { + providers = append(providers, auth.NewDisqus(makeParams(opts.DisqusCID, opts.DisqusCSEC))) } if len(providers) == 0 { log.Printf("[WARN] no auth providers defined") @@ -224,6 +209,7 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx // post-flush callback invoked by cache after each flush in async way func postFlushFn() { + // list of heavy urls for pre-heating on cache change urls := []string{ "http://localhost:%d/api/v1/list?site=%s", "http://localhost:%d/api/v1/last/50?site=%s", @@ -231,7 +217,7 @@ func postFlushFn() { for _, site := range opts.Sites { for _, u := range urls { - resp, err := http.Get(fmt.Sprintf(u, opts.ServerCommand.Port, site)) + resp, err := http.Get(fmt.Sprintf(u, opts.Port, site)) if err != nil { log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err) return diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index 736b6e4e..fd840dd0 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -53,6 +53,10 @@ type uid struct { // Import from disqus and save to store func (d *Disqus) Import(r io.Reader, siteID string) (err error) { + if err = d.DeleteAll(siteID); err != nil { + return err + } + commentsCh := d.convert(r, siteID) failed, passed := 0, 0 for c := range commentsCh { @@ -145,7 +149,6 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { log.Printf("[DEBUG] processed %d comments", stats.commentsCount) } } - } } close(commentsCh) diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go index ee59b8b9..41bf93aa 100644 --- a/app/migrator/migrator.go +++ b/app/migrator/migrator.go @@ -26,6 +26,7 @@ type Exporter interface { // CommentCreator is a minimal interface used by importer to make comments type CommentCreator interface { Create(comment store.Comment) (commentID string, err error) + DeleteAll(siteID string) error } // CommentFinder is a minimal interface used by exporter to find comments and list posts diff --git a/app/migrator/remark.go b/app/migrator/remark.go index 3923368b..f92aca04 100644 --- a/app/migrator/remark.go +++ b/app/migrator/remark.go @@ -56,6 +56,11 @@ func (r *Remark) Export(w io.Writer, siteID string) error { // Import comments from json strings produced by Remark.Export func (r *Remark) Import(reader io.Reader, siteID string) error { + + if err := r.DeleteAll(siteID); err != nil { + return err + } + failed := 0 total, comments := 0, 0 scanner := bufio.NewScanner(reader) diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 803a6f6d..b98eaf0a 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -20,7 +20,6 @@ import ( type admin struct { dataService store.Service exporter migrator.Exporter - importer migrator.Importer cache rest.LoadingCache defAvatarURL string } @@ -31,7 +30,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route router.Delete("/comment/{id}", a.deleteCommentCtrl) router.Put("/user/{userid}", a.setBlockCtrl) router.Get("/export", a.exportCtrl) - router.Post("/import", a.importCtrl) + router.Put("/pin/{id}", a.setPinCtrl) router.Get("/blocked", a.blockedUsersCtrl) return router @@ -112,16 +111,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { } } -// POST /import?site=site-id -// imports comments from post body. -func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) { - siteID := r.URL.Query().Get("site") - if err := a.importer.Import(r.Body, siteID); err != nil { - rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") - } - a.cache.Flush() -} - func (a *admin) checkBlocked(siteID string, user store.User) bool { return a.dataService.IsBlocked(siteID, user.ID) } diff --git a/app/rest/api/import.go b/app/rest/api/import.go new file mode 100644 index 00000000..314c3fa7 --- /dev/null +++ b/app/rest/api/import.go @@ -0,0 +1,59 @@ +package api + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/didip/tollbooth" + "github.com/didip/tollbooth_chi" + "github.com/go-chi/chi" + "github.com/go-chi/chi/middleware" + "github.com/umputun/remark/app/migrator" + "github.com/umputun/remark/app/rest" +) + +// Import rest runs on unexposed port and available for local requests only +type Import struct { + Version string + Cache rest.LoadingCache + NativeImporter migrator.Importer + DisqusImporter migrator.Importer + + httpServer *http.Server +} + +// Run the listener and request's router, activate rest server +// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way +func (s *Import) Run(port int) { + log.Printf("[INFO] activate import server on port %d", port) + + router := chi.NewRouter() + router.Use(middleware.RealIP, Recoverer) + router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) + router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil))) + router.Use(AppInfo("remark42-importer", s.Version), Ping, Logger(LogAll)) + + router.Post("/api/v1/admin/import", s.importCtrl) + + s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router} + err := s.httpServer.ListenAndServe() + log.Printf("[WARN] http server terminated, %s", err) +} + +// POST /import?site=site-id&provider=disqus|remark +// imports comments from post body. +func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site") + importer := s.NativeImporter + if r.URL.Query().Get("provider") == "disqus" { + importer = s.DisqusImporter + } + + if err := importer.Import(r.Body, siteID); err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") + return + } + s.Cache.Flush() +} diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go new file mode 100644 index 00000000..f4045547 --- /dev/null +++ b/app/rest/api/import_test.go @@ -0,0 +1,63 @@ +package api + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/app/migrator" + "github.com/umputun/remark/app/store" +) + +func TestImport(t *testing.T) { + srv, port := prepImportSrv(t) + assert.NotNil(t, srv) + defer cleanupImportSrv(srv) + + r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) + + resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native", + port), "application/json", r) + assert.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func prepImportSrv(t *testing.T) (srv *Import, port int) { + dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"}) + require.Nil(t, err) + + srv = &Import{ + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + Cache: &mockCache{}, + } + + portSetCh := make(chan bool) + + go func() { + port = rand.Intn(50000) + 1025 + portSetCh <- true + srv.Run(port) + }() + + <-portSetCh + + time.Sleep(100 * time.Millisecond) + return srv, port +} + +func cleanupImportSrv(srv *Import) { + srv.httpServer.Close() + srv.httpServer.Shutdown(context.Background()) + os.Remove(testDb) +} diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index 2036797c..67287a98 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -32,23 +32,27 @@ type Rest struct { Version string DataService store.Service Authenticator auth.Authenticator - Exporter migrator.Exporter Cache rest.LoadingCache WebRoot string + Exporter migrator.Exporter + httpServer *http.Server amdminService admin } // Run the lister and request's router, activate rest server func (s *Rest) Run(port int) { - log.Print("[INFO] activate rest server") + log.Printf("[INFO] activate rest server on port %d", port) if len(s.Authenticator.Admins) > 0 { log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins) } - s.amdminService = admin{dataService: s.DataService, exporter: s.Exporter, cache: s.Cache, + s.amdminService = admin{ + dataService: s.DataService, + exporter: s.Exporter, + cache: s.Cache, defAvatarURL: s.Authenticator.AvatarProxy.Default(), } diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index fa110ddb..714b6c9f 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -426,15 +426,27 @@ func prep(t *testing.T) (srv *Rest, port int) { WebRoot: "/tmp", } + importSrv := &Import{ + DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, + NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + Cache: &mockCache{}, + } + ioutil.WriteFile(testHTML, []byte("some html"), 0700) portSetCh := make(chan bool) + go func() { port = rand.Intn(50000) + 1025 portSetCh <- true srv.Run(port) }() + <-portSetCh + go func() { + importSrv.Run(port + 1) + }() + time.Sleep(100 * time.Millisecond) return srv, port } diff --git a/app/store/bolt.go b/app/store/bolt.go index 1904d181..fa9cc300 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -38,6 +38,8 @@ const ( userLimit = 50 ) +var topBuckets = []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName} + const tsNano = "2006-01-02T15:04:05.000000000Z07:00" // BoltSite defines single site param @@ -59,7 +61,6 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { // make top-level buckets err = db.Update(func(tx *bolt.Tx) error { - topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName} for _, bktName := range topBuckets { if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { return errors.Wrapf(err, "failed to create top level bucket %s", bktName) @@ -175,6 +176,29 @@ func (b *BoltDB) Delete(locator Locator, commentID string) error { }) } +// DeleteAll removes all top-level buckets for given siteID +func (b *BoltDB) DeleteAll(siteID string) error { + + bdb, err := b.db(siteID) + if err != nil { + return err + } + // delete top-level buckets + err = bdb.Update(func(tx *bolt.Tx) error { + for _, bktName := range topBuckets { + if e := tx.DeleteBucket([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to create top level bucket %s", bktName) + } + } + return nil + }) + + return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID) +} + // Find returns all comments for post and sorts results func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err error) { comments = []Comment{} diff --git a/app/store/bolt_test.go b/app/store/bolt_test.go index cb24de5a..7896f02a 100644 --- a/app/store/bolt_test.go +++ b/app/store/bolt_test.go @@ -52,6 +52,27 @@ func TestBoltDB_Delete(t *testing.T) { assert.Equal(t, 1, len(comments), "1 in last, 1 removed") } +func TestBoltDB_DeleteAll(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + err = b.DeleteAll("radio-t") + assert.Nil(t, err) + + comments, err := b.Last("radio-t", 10) + assert.Nil(t, err) + assert.Equal(t, 0, len(comments), "nothing left") + + c, err := b.Count(Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + assert.Nil(t, err) + assert.Equal(t, 0, c, "0 count") +} + func TestBoltDB_Get(t *testing.T) { defer os.Remove(testDb) b := prep(t) diff --git a/app/store/store.go b/app/store/store.go index bdb9a89f..e6e961d1 100644 --- a/app/store/store.go +++ b/app/store/store.go @@ -28,6 +28,7 @@ type Accessor interface { // Admin defines all store ops avail for admin only type Admin interface { Delete(locator Locator, commentID string) error // delete comment by id + DeleteAll(siteID string) error // delete all data from site SetBlock(siteID string, userID string, status bool) error // block or unblock user IsBlocked(siteID string, userID string) bool // check if user blocked Blocked(siteID string) ([]BlockedUser, error) // get list of blocked users diff --git a/import-disqus.sh b/import-disqus.sh new file mode 100644 index 00000000..cba716dc --- /dev/null +++ b/import-disqus.sh @@ -0,0 +1,4 @@ +#!/bin/sh +echo "import disqus file $1 to site $2" +curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark & +provider=disqus diff --git a/remark.rest b/remark.rest index ffaa988c..c125e6fb 100644 --- a/remark.rest +++ b/remark.rest @@ -103,4 +103,3 @@ GET {{host}}/api/v1/avatar/blah ### ping GET {{host}}/ping - diff --git a/start.sh b/start.sh index b8ee8fa0..3b87cb73 100755 --- a/start.sh +++ b/start.sh @@ -12,4 +12,4 @@ cp -nv /srv/web/remark.image /srv/var/avatars/remark.image echo "start remark42 server" -/sbin/su-exec ${USER} /srv/remark server +/sbin/su-exec ${USER} /srv/remark