Merge branch 'feature/import'
This commit is contained in:
Umputun
@@ -23,42 +23,37 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
|
||||
|
||||
* copy provided `docker-compose.yml` and customize for your needs
|
||||
* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
|
||||
* prepare user id for container ``export USER=`id -u $USER` ``
|
||||
* prepare user id for container `` export USER=`id -u $USER` ``
|
||||
* make sure you **don't keep** `DEV=true` for any non-development deployments
|
||||
* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
|
||||
* alternatively compile from sources - `docker-compose build`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Command line | Environment | Default | Multi | Scope | Description |
|
||||
| --------------- | -------------------- | ---------------------- | ----- | ------ | ------------------------------- |
|
||||
| --url | REMARK_URL | `https://remark42.com` | no | all | url to remark server |
|
||||
| --bolt | BOLTDB_PATH | `/tmp` | no | all | path to data directory |
|
||||
| --site | SITE | `remark` | yes | server | site name(s) |
|
||||
| --admin | ADMIN | | yes | server | admin names (list of user ids) |
|
||||
| --backup | BACKUP_PATH | `/tmp` | no | server | backups location |
|
||||
| --max-back | MAX_BACKUP_FILES | `10` | no | server | max backup files to keep |
|
||||
| --session | SESSION_STORE | `/tmp` | no | server | path to session store directory |
|
||||
| --store-key | STORE_KEY | `secure-store-key` | no | server | session store encryption key |
|
||||
| --google-cid | REMARK_GOOGLE_CID | | no | server | Google OAuth client ID |
|
||||
| --google-csec | REMARK_GOOGLE_CSEC | | no | server | Google OAuth client secret |
|
||||
| --facebook-cid | REMARK_FACEBOOK_CID | | no | server | Facebook OAuth client ID |
|
||||
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | server | Facebook OAuth client secret |
|
||||
| --github-cid | REMARK_GITHUB_CID | | no | server | Github OAuth client ID |
|
||||
| --github-csec | REMARK_GITHUB_CSEC | | no | server | Github OAuth client secret |
|
||||
| --provider | | `disqus` | no | import | provider type for import |
|
||||
| --site | | `remark` | no | import | site ID |
|
||||
| --file | | `disqus.xml` | no | import | import file |
|
||||
| --dbg | DEBUG | `false` | no | all | debug mode |
|
||||
| --dev-password | DEV_PASSWD | | no | all | password for `dev` user |
|
||||
| Command line | Environment | Default | Multi | Description |
|
||||
| --------------- | -------------------- | ---------------------- | ----- | ------------------------------- |
|
||||
| --url | REMARK_URL | `https://remark42.com` | no | url to remark server |
|
||||
| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory |
|
||||
| --site | SITE | `remark` | yes | site name(s) |
|
||||
| --admin | ADMIN | | yes | admin names (list of user ids) |
|
||||
| --backup | BACKUP_PATH | `/tmp` | no | backups location |
|
||||
| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep |
|
||||
| --session | SESSION_STORE | `/tmp` | no | path to session store directory |
|
||||
| --store-key | STORE_KEY | `secure-store-key` | no | session store encryption key |
|
||||
| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID |
|
||||
| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret |
|
||||
| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID |
|
||||
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret |
|
||||
| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID |
|
||||
| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret |
|
||||
| --provider | | `disqus` | no | provider type for import |
|
||||
| --site | | `remark` | no | site ID |
|
||||
| --file | | `disqus.xml` | no | import file |
|
||||
| --dbg | DEBUG | `false` | no | debug mode |
|
||||
| --dev-password | DEV_PASSWD | | no | password for `dev` user |
|
||||
|
||||
_all multi parameters separated by `,`_
|
||||
|
||||
#### Run modes
|
||||
|
||||
* `server` activates regular, server mode
|
||||
* `import` performs import from external providers (disqus and internal json, see `/api/v1/admin/export`)
|
||||
|
||||
#### Register oauth2 providers
|
||||
|
||||
Authentication handled by external providers. You should setup oauth2 for all (or some) of them in order to allow users to make comments. It is not mandatory to have all of them, but at least one should be property configured.
|
||||
@@ -101,10 +96,8 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
|
||||
#### Initial import from Disqus
|
||||
|
||||
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
|
||||
2. Move this file to your remark42 host and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
|
||||
3. Stop remark42 containers if started, i.e. `docker-compose stop`
|
||||
4. Run import command - `docker-compose run --rm remark /srv/remark import --file=<disqus-export-name>.xml --site=<your site id>`
|
||||
5. Start remark42 containers `docker-compose up -d`
|
||||
2. Move this file to your remark42 host within `.var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
|
||||
3. Run import command - `docker-compose run remark /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
|
||||
|
||||
### Frontend
|
||||
|
||||
@@ -112,20 +105,20 @@ Frontend part is building automatically along with backend if you use `docker-co
|
||||
|
||||
For manual building:
|
||||
|
||||
- install [Node.js 8](https://nodejs.org/en/) or higher;
|
||||
- run `npm install` inside `./web`;
|
||||
- run `npm run build` there;
|
||||
- result files will be saved in `./web/public`.
|
||||
* install [Node.js 8](https://nodejs.org/en/) or higher;
|
||||
* run `npm install` inside `./web`;
|
||||
* run `npm run build` there;
|
||||
* result files will be saved in `./web/public`.
|
||||
|
||||
For development mode use `npm start` instead of `npm run build`.
|
||||
In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
|
||||
|
||||
URLs for development:
|
||||
|
||||
- `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
|
||||
- `localhost:8080/dev.html` — page with embedded script from local folder;
|
||||
- `localhost:8080/last-comments.html` — page with embedded script for last comments;
|
||||
- `localhost:8080/counter.html` — page with embedded script for counter with examples.
|
||||
* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
|
||||
* `localhost:8080/dev.html` — page with embedded script from local folder;
|
||||
* `localhost:8080/last-comments.html` — page with embedded script for last comments;
|
||||
* `localhost:8080/counter.html` — page with embedded script for counter with examples.
|
||||
|
||||
## API
|
||||
|
||||
|
||||
+45
-59
@@ -33,30 +33,22 @@ var opts struct {
|
||||
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var" description:"backups location"`
|
||||
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
|
||||
|
||||
ServerCommand struct {
|
||||
SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
|
||||
StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
|
||||
SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
|
||||
StoreKey string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
|
||||
|
||||
GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
|
||||
GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
|
||||
GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
|
||||
GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
|
||||
FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
|
||||
FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
|
||||
DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
|
||||
DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
|
||||
GoogleCID string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
|
||||
GoogleCSEC string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
|
||||
GithubCID string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
|
||||
GithubCSEC string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
|
||||
FacebookCID string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
|
||||
FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
|
||||
DisqusCID string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
|
||||
DisqusCSEC string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
|
||||
|
||||
AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
|
||||
DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
|
||||
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
|
||||
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
|
||||
} `command:"server" description:"run server"`
|
||||
|
||||
ImportCommand struct {
|
||||
Provider string `long:"provider" default:"disqus" description:"provider type"`
|
||||
SiteID string `long:"site" default:"remark" description:"site ID"`
|
||||
InputFile string `long:"file" default:"disqus.xml" description:"input file"`
|
||||
} `command:"import" description:"import comments from external sources"`
|
||||
AvatarStore string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
|
||||
DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
|
||||
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
|
||||
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
|
||||
}
|
||||
|
||||
var revision = "unknown"
|
||||
@@ -71,33 +63,19 @@ func main() {
|
||||
setupLog(opts.Dbg)
|
||||
log.Print("[INFO] started remark")
|
||||
|
||||
if err := makeDirs(opts.BoltPath, opts.ServerCommand.SessionStore, opts.BackupLocation, opts.ServerCommand.AvatarStore); err != nil {
|
||||
if err := makeDirs(opts.BoltPath, opts.SessionStore, opts.BackupLocation, opts.AvatarStore); err != nil {
|
||||
log.Fatalf("[ERROR] can't create directories, %+v", err)
|
||||
}
|
||||
|
||||
dataStore := makeBoltStore(opts.Sites)
|
||||
|
||||
if p.Active != nil && p.Command.Find("import") == p.Active {
|
||||
// import mode
|
||||
params := migrator.ImportParams{
|
||||
CommentCreator: dataStore,
|
||||
InputFile: opts.ImportCommand.InputFile,
|
||||
Provider: opts.ImportCommand.Provider,
|
||||
SiteID: opts.ImportCommand.SiteID,
|
||||
}
|
||||
if err := migrator.ImportComments(params); err != nil {
|
||||
log.Fatalf("[ERROR] failed to import, %+v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if opts.DevPasswd != "" {
|
||||
log.Printf("[WARN] running in dev mode")
|
||||
}
|
||||
|
||||
dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.ServerCommand.StoreKey}
|
||||
dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.StoreKey}
|
||||
sessionStore := func() sessions.Store {
|
||||
sess := sessions.NewFilesystemStore(opts.ServerCommand.SessionStore, []byte(opts.ServerCommand.StoreKey))
|
||||
sess := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey))
|
||||
sess.Options.HttpOnly = true
|
||||
sess.Options.Secure = true
|
||||
sess.Options.MaxAge = 3600 * 24 * 365
|
||||
@@ -105,22 +83,31 @@ func main() {
|
||||
return sess
|
||||
}()
|
||||
|
||||
exporter := &migrator.Remark{CommentFinder: dataStore}
|
||||
exporter := migrator.Remark{CommentFinder: dataStore}
|
||||
cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn)
|
||||
|
||||
activateBackup(&exporter)
|
||||
|
||||
importSrv := api.Import{
|
||||
Version: revision,
|
||||
Cache: cache,
|
||||
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
|
||||
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
|
||||
}
|
||||
go importSrv.Run(opts.Port + 1)
|
||||
|
||||
avatarProxy := &auth.AvatarProxy{
|
||||
StorePath: opts.ServerCommand.AvatarStore,
|
||||
StorePath: opts.AvatarStore,
|
||||
RoutePath: "/api/v1/avatar",
|
||||
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
|
||||
DefaultAvatar: opts.ServerCommand.DefaultAvatar,
|
||||
DefaultAvatar: opts.DefaultAvatar,
|
||||
}
|
||||
|
||||
activateBackup(exporter)
|
||||
|
||||
srv := api.Rest{
|
||||
Version: revision,
|
||||
DataService: dataService,
|
||||
Exporter: exporter,
|
||||
WebRoot: opts.ServerCommand.WebRoot,
|
||||
Exporter: &exporter,
|
||||
WebRoot: opts.WebRoot,
|
||||
Authenticator: auth.Authenticator{
|
||||
Admins: opts.Admins,
|
||||
SessionStore: sessionStore,
|
||||
@@ -128,9 +115,9 @@ func main() {
|
||||
AvatarProxy: avatarProxy,
|
||||
DevPasswd: opts.DevPasswd,
|
||||
},
|
||||
Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn),
|
||||
Cache: cache,
|
||||
}
|
||||
srv.Run(opts.ServerCommand.Port)
|
||||
srv.Run(opts.Port)
|
||||
}
|
||||
|
||||
// activateBackup runs background backups for each site
|
||||
@@ -201,19 +188,17 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
|
||||
}
|
||||
}
|
||||
|
||||
srvOpts := opts.ServerCommand
|
||||
|
||||
if srvOpts.GoogleCID != "" && srvOpts.GoogleCSEC != "" {
|
||||
providers = append(providers, auth.NewGoogle(makeParams(srvOpts.GoogleCID, srvOpts.GoogleCSEC)))
|
||||
if opts.GoogleCID != "" && opts.GoogleCSEC != "" {
|
||||
providers = append(providers, auth.NewGoogle(makeParams(opts.GoogleCID, opts.GoogleCSEC)))
|
||||
}
|
||||
if srvOpts.GithubCID != "" && srvOpts.GithubCSEC != "" {
|
||||
providers = append(providers, auth.NewGithub(makeParams(srvOpts.GithubCID, srvOpts.GithubCSEC)))
|
||||
if opts.GithubCID != "" && opts.GithubCSEC != "" {
|
||||
providers = append(providers, auth.NewGithub(makeParams(opts.GithubCID, opts.GithubCSEC)))
|
||||
}
|
||||
if srvOpts.FacebookCID != "" && srvOpts.FacebookCSEC != "" {
|
||||
providers = append(providers, auth.NewFacebook(makeParams(srvOpts.FacebookCID, srvOpts.FacebookCSEC)))
|
||||
if opts.FacebookCID != "" && opts.FacebookCSEC != "" {
|
||||
providers = append(providers, auth.NewFacebook(makeParams(opts.FacebookCID, opts.FacebookCSEC)))
|
||||
}
|
||||
if srvOpts.DisqusCID != "" && srvOpts.DisqusCSEC != "" {
|
||||
providers = append(providers, auth.NewDisqus(makeParams(srvOpts.DisqusCID, srvOpts.DisqusCSEC)))
|
||||
if opts.DisqusCID != "" && opts.DisqusCSEC != "" {
|
||||
providers = append(providers, auth.NewDisqus(makeParams(opts.DisqusCID, opts.DisqusCSEC)))
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
log.Printf("[WARN] no auth providers defined")
|
||||
@@ -224,6 +209,7 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
|
||||
// post-flush callback invoked by cache after each flush in async way
|
||||
func postFlushFn() {
|
||||
|
||||
// list of heavy urls for pre-heating on cache change
|
||||
urls := []string{
|
||||
"http://localhost:%d/api/v1/list?site=%s",
|
||||
"http://localhost:%d/api/v1/last/50?site=%s",
|
||||
@@ -231,7 +217,7 @@ func postFlushFn() {
|
||||
|
||||
for _, site := range opts.Sites {
|
||||
for _, u := range urls {
|
||||
resp, err := http.Get(fmt.Sprintf(u, opts.ServerCommand.Port, site))
|
||||
resp, err := http.Get(fmt.Sprintf(u, opts.Port, site))
|
||||
if err != nil {
|
||||
log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
|
||||
return
|
||||
|
||||
@@ -53,6 +53,10 @@ type uid struct {
|
||||
// Import from disqus and save to store
|
||||
func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
|
||||
|
||||
if err = d.DeleteAll(siteID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
commentsCh := d.convert(r, siteID)
|
||||
failed, passed := 0, 0
|
||||
for c := range commentsCh {
|
||||
@@ -145,7 +149,6 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
log.Printf("[DEBUG] processed %d comments", stats.commentsCount)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
close(commentsCh)
|
||||
|
||||
@@ -26,6 +26,7 @@ type Exporter interface {
|
||||
// CommentCreator is a minimal interface used by importer to make comments
|
||||
type CommentCreator interface {
|
||||
Create(comment store.Comment) (commentID string, err error)
|
||||
DeleteAll(siteID string) error
|
||||
}
|
||||
|
||||
// CommentFinder is a minimal interface used by exporter to find comments and list posts
|
||||
|
||||
@@ -56,6 +56,11 @@ func (r *Remark) Export(w io.Writer, siteID string) error {
|
||||
|
||||
// Import comments from json strings produced by Remark.Export
|
||||
func (r *Remark) Import(reader io.Reader, siteID string) error {
|
||||
|
||||
if err := r.DeleteAll(siteID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
failed := 0
|
||||
total, comments := 0, 0
|
||||
scanner := bufio.NewScanner(reader)
|
||||
|
||||
+1
-12
@@ -20,7 +20,6 @@ import (
|
||||
type admin struct {
|
||||
dataService store.Service
|
||||
exporter migrator.Exporter
|
||||
importer migrator.Importer
|
||||
cache rest.LoadingCache
|
||||
defAvatarURL string
|
||||
}
|
||||
@@ -31,7 +30,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
|
||||
router.Delete("/comment/{id}", a.deleteCommentCtrl)
|
||||
router.Put("/user/{userid}", a.setBlockCtrl)
|
||||
router.Get("/export", a.exportCtrl)
|
||||
router.Post("/import", a.importCtrl)
|
||||
|
||||
router.Put("/pin/{id}", a.setPinCtrl)
|
||||
router.Get("/blocked", a.blockedUsersCtrl)
|
||||
return router
|
||||
@@ -112,16 +111,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// POST /import?site=site-id
|
||||
// imports comments from post body.
|
||||
func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
siteID := r.URL.Query().Get("site")
|
||||
if err := a.importer.Import(r.Body, siteID); err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
|
||||
}
|
||||
a.cache.Flush()
|
||||
}
|
||||
|
||||
func (a *admin) checkBlocked(siteID string, user store.User) bool {
|
||||
return a.dataService.IsBlocked(siteID, user.ID)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/didip/tollbooth"
|
||||
"github.com/didip/tollbooth_chi"
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/go-chi/chi/middleware"
|
||||
"github.com/umputun/remark/app/migrator"
|
||||
"github.com/umputun/remark/app/rest"
|
||||
)
|
||||
|
||||
// Import rest runs on unexposed port and available for local requests only
|
||||
type Import struct {
|
||||
Version string
|
||||
Cache rest.LoadingCache
|
||||
NativeImporter migrator.Importer
|
||||
DisqusImporter migrator.Importer
|
||||
|
||||
httpServer *http.Server
|
||||
}
|
||||
|
||||
// Run the listener and request's router, activate rest server
|
||||
// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
|
||||
func (s *Import) Run(port int) {
|
||||
log.Printf("[INFO] activate import server on port %d", port)
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Use(middleware.RealIP, Recoverer)
|
||||
router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
|
||||
router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
|
||||
router.Use(AppInfo("remark42-importer", s.Version), Ping, Logger(LogAll))
|
||||
|
||||
router.Post("/api/v1/admin/import", s.importCtrl)
|
||||
|
||||
s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
|
||||
err := s.httpServer.ListenAndServe()
|
||||
log.Printf("[WARN] http server terminated, %s", err)
|
||||
}
|
||||
|
||||
// POST /import?site=site-id&provider=disqus|remark
|
||||
// imports comments from post body.
|
||||
func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
siteID := r.URL.Query().Get("site")
|
||||
importer := s.NativeImporter
|
||||
if r.URL.Query().Get("provider") == "disqus" {
|
||||
importer = s.DisqusImporter
|
||||
}
|
||||
|
||||
if err := importer.Import(r.Body, siteID); err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
|
||||
return
|
||||
}
|
||||
s.Cache.Flush()
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/bbolt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark/app/migrator"
|
||||
"github.com/umputun/remark/app/store"
|
||||
)
|
||||
|
||||
func TestImport(t *testing.T) {
|
||||
srv, port := prepImportSrv(t)
|
||||
assert.NotNil(t, srv)
|
||||
defer cleanupImportSrv(srv)
|
||||
|
||||
r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
|
||||
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||||
|
||||
resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
|
||||
port), "application/json", r)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
func prepImportSrv(t *testing.T) (srv *Import, port int) {
|
||||
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"})
|
||||
require.Nil(t, err)
|
||||
|
||||
srv = &Import{
|
||||
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
|
||||
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
|
||||
Cache: &mockCache{},
|
||||
}
|
||||
|
||||
portSetCh := make(chan bool)
|
||||
|
||||
go func() {
|
||||
port = rand.Intn(50000) + 1025
|
||||
portSetCh <- true
|
||||
srv.Run(port)
|
||||
}()
|
||||
|
||||
<-portSetCh
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return srv, port
|
||||
}
|
||||
|
||||
func cleanupImportSrv(srv *Import) {
|
||||
srv.httpServer.Close()
|
||||
srv.httpServer.Shutdown(context.Background())
|
||||
os.Remove(testDb)
|
||||
}
|
||||
@@ -32,23 +32,27 @@ type Rest struct {
|
||||
Version string
|
||||
DataService store.Service
|
||||
Authenticator auth.Authenticator
|
||||
Exporter migrator.Exporter
|
||||
Cache rest.LoadingCache
|
||||
WebRoot string
|
||||
|
||||
Exporter migrator.Exporter
|
||||
|
||||
httpServer *http.Server
|
||||
amdminService admin
|
||||
}
|
||||
|
||||
// Run the lister and request's router, activate rest server
|
||||
func (s *Rest) Run(port int) {
|
||||
log.Print("[INFO] activate rest server")
|
||||
log.Printf("[INFO] activate rest server on port %d", port)
|
||||
|
||||
if len(s.Authenticator.Admins) > 0 {
|
||||
log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
|
||||
}
|
||||
|
||||
s.amdminService = admin{dataService: s.DataService, exporter: s.Exporter, cache: s.Cache,
|
||||
s.amdminService = admin{
|
||||
dataService: s.DataService,
|
||||
exporter: s.Exporter,
|
||||
cache: s.Cache,
|
||||
defAvatarURL: s.Authenticator.AvatarProxy.Default(),
|
||||
}
|
||||
|
||||
|
||||
@@ -426,15 +426,27 @@ func prep(t *testing.T) (srv *Rest, port int) {
|
||||
WebRoot: "/tmp",
|
||||
}
|
||||
|
||||
importSrv := &Import{
|
||||
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
|
||||
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
|
||||
Cache: &mockCache{},
|
||||
}
|
||||
|
||||
ioutil.WriteFile(testHTML, []byte("some html"), 0700)
|
||||
portSetCh := make(chan bool)
|
||||
|
||||
go func() {
|
||||
port = rand.Intn(50000) + 1025
|
||||
portSetCh <- true
|
||||
srv.Run(port)
|
||||
}()
|
||||
|
||||
<-portSetCh
|
||||
|
||||
go func() {
|
||||
importSrv.Run(port + 1)
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return srv, port
|
||||
}
|
||||
|
||||
+25
-1
@@ -38,6 +38,8 @@ const (
|
||||
userLimit = 50
|
||||
)
|
||||
|
||||
var topBuckets = []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
|
||||
|
||||
const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
|
||||
// BoltSite defines single site param
|
||||
@@ -59,7 +61,6 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
|
||||
|
||||
// make top-level buckets
|
||||
err = db.Update(func(tx *bolt.Tx) error {
|
||||
topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
|
||||
for _, bktName := range topBuckets {
|
||||
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
|
||||
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
|
||||
@@ -175,6 +176,29 @@ func (b *BoltDB) Delete(locator Locator, commentID string) error {
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteAll removes all top-level buckets for given siteID
|
||||
func (b *BoltDB) DeleteAll(siteID string) error {
|
||||
|
||||
bdb, err := b.db(siteID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// delete top-level buckets
|
||||
err = bdb.Update(func(tx *bolt.Tx) error {
|
||||
for _, bktName := range topBuckets {
|
||||
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
|
||||
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
|
||||
}
|
||||
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
|
||||
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
|
||||
}
|
||||
|
||||
// Find returns all comments for post and sorts results
|
||||
func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err error) {
|
||||
comments = []Comment{}
|
||||
|
||||
@@ -52,6 +52,27 @@ func TestBoltDB_Delete(t *testing.T) {
|
||||
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
|
||||
}
|
||||
|
||||
func TestBoltDB_DeleteAll(t *testing.T) {
|
||||
defer os.Remove(testDb)
|
||||
b := prep(t)
|
||||
|
||||
loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
|
||||
res, err := b.Find(loc, "time")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(res), "initially 2 comments")
|
||||
|
||||
err = b.DeleteAll("radio-t")
|
||||
assert.Nil(t, err)
|
||||
|
||||
comments, err := b.Last("radio-t", 10)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(comments), "nothing left")
|
||||
|
||||
c, err := b.Count(Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, c, "0 count")
|
||||
}
|
||||
|
||||
func TestBoltDB_Get(t *testing.T) {
|
||||
defer os.Remove(testDb)
|
||||
b := prep(t)
|
||||
|
||||
@@ -28,6 +28,7 @@ type Accessor interface {
|
||||
// Admin defines all store ops avail for admin only
|
||||
type Admin interface {
|
||||
Delete(locator Locator, commentID string) error // delete comment by id
|
||||
DeleteAll(siteID string) error // delete all data from site
|
||||
SetBlock(siteID string, userID string, status bool) error // block or unblock user
|
||||
IsBlocked(siteID string, userID string) bool // check if user blocked
|
||||
Blocked(siteID string) ([]BlockedUser, error) // get list of blocked users
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
echo "import disqus file $1 to site $2"
|
||||
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark &
|
||||
provider=disqus
|
||||
@@ -103,4 +103,3 @@ GET {{host}}/api/v1/avatar/blah
|
||||
|
||||
### ping
|
||||
GET {{host}}/ping
|
||||
|
||||
|
||||
Reference in New Issue
Block a user