From 9b9479367ad1b217efe15b1c69f6e80e2098f096 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 30 Apr 2018 00:54:15 -0500
Subject: [PATCH 1/4] in-proc import with initial delete

---
 app/main.go              |  2 ++
 app/migrator/disqus.go   |  5 ++++-
 app/migrator/migrator.go |  1 +
 app/migrator/remark.go   |  5 +++++
 app/rest/api/admin.go    | 22 ++++++++++++++--------
 app/rest/api/rest.go     | 14 +++++++++++---
 app/store/bolt.go        | 26 +++++++++++++++++++++++++-
 app/store/bolt_test.go   | 21 +++++++++++++++++++++
 app/store/store.go       |  1 +
 9 files changed, 84 insertions(+), 13 deletions(-)

diff --git a/app/main.go b/app/main.go
index 333b7e38..e3a87156 100644
--- a/app/main.go
+++ b/app/main.go
@@ -120,6 +120,8 @@ func main() {
 		Version:     revision,
 		DataService: dataService,
 		Exporter:    exporter,
+		Importer:    &migrator.Remark{CommentCreator: dataStore},
+		Disqus:      &migrator.Disqus{CommentCreator: dataStore},
 		WebRoot:     opts.ServerCommand.WebRoot,
 		Authenticator: auth.Authenticator{
 			Admins:       opts.Admins,
diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go
index 736b6e4e..fd840dd0 100644
--- a/app/migrator/disqus.go
+++ b/app/migrator/disqus.go
@@ -53,6 +53,10 @@ type uid struct {
 // Import from disqus and save to store
 func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
 
+	if err = d.DeleteAll(siteID); err != nil {
+		return err
+	}
+
 	commentsCh := d.convert(r, siteID)
 	failed, passed := 0, 0
 	for c := range commentsCh {
@@ -145,7 +149,6 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 						log.Printf("[DEBUG] processed %d comments", stats.commentsCount)
 					}
 				}
-
 			}
 		}
 		close(commentsCh)
diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go
index ee59b8b9..41bf93aa 100644
--- a/app/migrator/migrator.go
+++ b/app/migrator/migrator.go
@@ -26,6 +26,7 @@ type Exporter interface {
 // CommentCreator is a minimal interface used by importer to make comments
 type CommentCreator interface {
 	Create(comment store.Comment) (commentID string, err error)
+	DeleteAll(siteID string) error
 }
 
 // CommentFinder is a minimal interface used by exporter to find comments and list posts
diff --git a/app/migrator/remark.go b/app/migrator/remark.go
index 3923368b..f92aca04 100644
--- a/app/migrator/remark.go
+++ b/app/migrator/remark.go
@@ -56,6 +56,11 @@ func (r *Remark) Export(w io.Writer, siteID string) error {
 
 // Import comments from json strings produced by Remark.Export
 func (r *Remark) Import(reader io.Reader, siteID string) error {
+
+	if err := r.DeleteAll(siteID); err != nil {
+		return err
+	}
+
 	failed := 0
 	total, comments := 0, 0
 	scanner := bufio.NewScanner(reader)
diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go
index 803a6f6d..8eada1e8 100644
--- a/app/rest/api/admin.go
+++ b/app/rest/api/admin.go
@@ -18,11 +18,12 @@ import (
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService  store.Service
-	exporter     migrator.Exporter
-	importer     migrator.Importer
-	cache        rest.LoadingCache
-	defAvatarURL string
+	dataService    store.Service
+	exporterNative migrator.Exporter
+	importerNative migrator.Importer
+	importerDisqus migrator.Importer
+	cache          rest.LoadingCache
+	defAvatarURL   string
 }
 
 func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
@@ -107,16 +108,21 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		writer = gzip.NewWriter(w)
 	}
 
-	if err := a.exporter.Export(writer, siteID); err != nil {
+	if err := a.exporterNative.Export(writer, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
 	}
 }
 
-// POST /import?site=site-id
+// POST /import?site=site-id&provider=disqus|remark
 // imports comments from post body.
 func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	if err := a.importer.Import(r.Body, siteID); err != nil {
+	importer := a.importerNative
+	if r.URL.Query().Get("provider") == "disqus" {
+		importer = a.importerDisqus
+	}
+
+	if err := importer.Import(r.Body, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
 	}
 	a.cache.Flush()
diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go
index 2036797c..bc7822d3 100644
--- a/app/rest/api/rest.go
+++ b/app/rest/api/rest.go
@@ -32,10 +32,13 @@ type Rest struct {
 	Version       string
 	DataService   store.Service
 	Authenticator auth.Authenticator
-	Exporter      migrator.Exporter
 	Cache         rest.LoadingCache
 	WebRoot       string
 
+	Exporter migrator.Exporter
+	Importer migrator.Importer
+	Disqus   migrator.Importer
+
 	httpServer    *http.Server
 	amdminService admin
 }
@@ -48,8 +51,13 @@ func (s *Rest) Run(port int) {
 		log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
 	}
 
-	s.amdminService = admin{dataService: s.DataService, exporter: s.Exporter, cache: s.Cache,
-		defAvatarURL: s.Authenticator.AvatarProxy.Default(),
+	s.amdminService = admin{
+		dataService:    s.DataService,
+		exporterNative: s.Exporter,
+		importerNative: s.Importer,
+		importerDisqus: s.Disqus,
+		cache:          s.Cache,
+		defAvatarURL:   s.Authenticator.AvatarProxy.Default(),
 	}
 
 	router := chi.NewRouter()
diff --git a/app/store/bolt.go b/app/store/bolt.go
index 1904d181..fa9cc300 100644
--- a/app/store/bolt.go
+++ b/app/store/bolt.go
@@ -38,6 +38,8 @@ const (
 	userLimit = 50
 )
 
+var topBuckets = []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
+
 const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
 
 // BoltSite defines single site param
@@ -59,7 +61,6 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 
 		// make top-level buckets
 		err = db.Update(func(tx *bolt.Tx) error {
-			topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
 			for _, bktName := range topBuckets {
 				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
 					return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
@@ -175,6 +176,29 @@ func (b *BoltDB) Delete(locator Locator, commentID string) error {
 	})
 }
 
+// DeleteAll removes all top-level buckets for given siteID
+func (b *BoltDB) DeleteAll(siteID string) error {
+
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return err
+	}
+	// delete top-level buckets
+	err = bdb.Update(func(tx *bolt.Tx) error {
+		for _, bktName := range topBuckets {
+			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
+				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
+			}
+			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
+			}
+		}
+		return nil
+	})
+
+	return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
+}
+
 // Find returns all comments for post and sorts results
 func (b *BoltDB) Find(locator Locator, sortFld string) (comments []Comment, err error) {
 	comments = []Comment{}
diff --git a/app/store/bolt_test.go b/app/store/bolt_test.go
index cb24de5a..7896f02a 100644
--- a/app/store/bolt_test.go
+++ b/app/store/bolt_test.go
@@ -52,6 +52,27 @@ func TestBoltDB_Delete(t *testing.T) {
 	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
 }
 
+func TestBoltDB_DeleteAll(t *testing.T) {
+	defer os.Remove(testDb)
+	b := prep(t)
+
+	loc := Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+	res, err := b.Find(loc, "time")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	err = b.DeleteAll("radio-t")
+	assert.Nil(t, err)
+
+	comments, err := b.Last("radio-t", 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	c, err := b.Count(Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+	assert.Nil(t, err)
+	assert.Equal(t, 0, c, "0 count")
+}
+
 func TestBoltDB_Get(t *testing.T) {
 	defer os.Remove(testDb)
 	b := prep(t)
diff --git a/app/store/store.go b/app/store/store.go
index bdb9a89f..e6e961d1 100644
--- a/app/store/store.go
+++ b/app/store/store.go
@@ -28,6 +28,7 @@ type Accessor interface {
 // Admin defines all store ops avail for admin only
 type Admin interface {
 	Delete(locator Locator, commentID string) error           // delete comment by id
+	DeleteAll(siteID string) error                            // delete all data from site
 	SetBlock(siteID string, userID string, status bool) error // block or unblock  user
 	IsBlocked(siteID string, userID string) bool              // check if user blocked
 	Blocked(siteID string) ([]BlockedUser, error)             // get list of blocked users

From 15095a915150b05c9f11ae024fca3dc4150b0597 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 30 Apr 2018 02:14:14 -0500
Subject: [PATCH 2/4] remove import cmd, add import test

---
 app/main.go                | 98 ++++++++++++++------------------------
 app/rest/api/admin.go      | 12 ++---
 app/rest/api/admin_test.go | 29 +++++++++++
 app/rest/api/rest.go       | 10 ++--
 app/rest/api/rest_test.go  |  7 +--
 remark.rest                |  1 -
 start.sh                   |  2 +-
 7 files changed, 81 insertions(+), 78 deletions(-)

diff --git a/app/main.go b/app/main.go
index e3a87156..06b595df 100644
--- a/app/main.go
+++ b/app/main.go
@@ -33,30 +33,22 @@ var opts struct {
 	BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var" description:"backups location"`
 	MaxBackupFiles int    `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
 
-	ServerCommand struct {
-		SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
-		StoreKey     string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
+	SessionStore string `long:"session" env:"SESSION_STORE" default:"./var" description:"path to session store directory"`
+	StoreKey     string `long:"store-key" env:"STORE_KEY" default:"secure-store-key" description:"store key"`
 
-		GoogleCID    string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
-		GoogleCSEC   string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
-		GithubCID    string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
-		GithubCSEC   string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
-		FacebookCID  string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
-		FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
-		DisqusCID    string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
-		DisqusCSEC   string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
+	GoogleCID    string `long:"google-cid" env:"REMARK_GOOGLE_CID" description:"Google OAuth client ID"`
+	GoogleCSEC   string `long:"google-csec" env:"REMARK_GOOGLE_CSEC" description:"Google OAuth client secret"`
+	GithubCID    string `long:"github-cid" env:"REMARK_GITHUB_CID" description:"Github OAuth client ID"`
+	GithubCSEC   string `long:"github-csec" env:"REMARK_GITHUB_CSEC" description:"Github OAuth client secret"`
+	FacebookCID  string `long:"facebook-cid" env:"REMARK_FACEBOOK_CID" description:"Facebook OAuth client ID"`
+	FacebookCSEC string `long:"facebook-csec" env:"REMARK_FACEBOOK_CSEC" description:"Facebook OAuth client secret"`
+	DisqusCID    string `long:"disqus-cid" env:"REMARK_DISQUS_CID" description:"Disqus OAuth client ID"`
+	DisqusCSEC   string `long:"disqus-csec" env:"REMARK_DISQUS_CSEC" description:"Disqus OAuth client secret"`
 
-		AvatarStore   string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
-		DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
-		Port          int    `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-		WebRoot       string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	} `command:"server" description:"run server"`
-
-	ImportCommand struct {
-		Provider  string `long:"provider" default:"disqus" description:"provider type"`
-		SiteID    string `long:"site" default:"remark" description:"site ID"`
-		InputFile string `long:"file" default:"disqus.xml" description:"input file"`
-	} `command:"import" description:"import comments from external sources"`
+	AvatarStore   string `long:"avatars" env:"AVATAR_STORE" default:"./var/avatars" description:"path to avatars directory"`
+	DefaultAvatar string `long:"avatar-def" env:"AVATAR_DEF" default:"remark.image" description:"default avatar"`
+	Port          int    `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	WebRoot       string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 }
 
 var revision = "unknown"
@@ -71,33 +63,19 @@ func main() {
 	setupLog(opts.Dbg)
 	log.Print("[INFO] started remark")
 
-	if err := makeDirs(opts.BoltPath, opts.ServerCommand.SessionStore, opts.BackupLocation, opts.ServerCommand.AvatarStore); err != nil {
+	if err := makeDirs(opts.BoltPath, opts.SessionStore, opts.BackupLocation, opts.AvatarStore); err != nil {
 		log.Fatalf("[ERROR] can't create directories, %+v", err)
 	}
 
 	dataStore := makeBoltStore(opts.Sites)
 
-	if p.Active != nil && p.Command.Find("import") == p.Active {
-		// import mode
-		params := migrator.ImportParams{
-			CommentCreator: dataStore,
-			InputFile:      opts.ImportCommand.InputFile,
-			Provider:       opts.ImportCommand.Provider,
-			SiteID:         opts.ImportCommand.SiteID,
-		}
-		if err := migrator.ImportComments(params); err != nil {
-			log.Fatalf("[ERROR] failed to import, %+v", err)
-		}
-		return
-	}
-
 	if opts.DevPasswd != "" {
 		log.Printf("[WARN] running in dev mode")
 	}
 
-	dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.ServerCommand.StoreKey}
+	dataService := store.Service{Interface: dataStore, EditDuration: 5 * time.Minute, Secret: opts.StoreKey}
 	sessionStore := func() sessions.Store {
-		sess := sessions.NewFilesystemStore(opts.ServerCommand.SessionStore, []byte(opts.ServerCommand.StoreKey))
+		sess := sessions.NewFilesystemStore(opts.SessionStore, []byte(opts.StoreKey))
 		sess.Options.HttpOnly = true
 		sess.Options.Secure = true
 		sess.Options.MaxAge = 3600 * 24 * 365
@@ -105,24 +83,23 @@ func main() {
 		return sess
 	}()
 
-	exporter := &migrator.Remark{CommentFinder: dataStore}
+	nativeMigrator := migrator.Remark{CommentFinder: dataStore}
 
 	avatarProxy := &auth.AvatarProxy{
-		StorePath:     opts.ServerCommand.AvatarStore,
+		StorePath:     opts.AvatarStore,
 		RoutePath:     "/api/v1/avatar",
 		RemarkURL:     strings.TrimSuffix(opts.RemarkURL, "/"),
-		DefaultAvatar: opts.ServerCommand.DefaultAvatar,
+		DefaultAvatar: opts.DefaultAvatar,
 	}
 
-	activateBackup(exporter)
+	activateBackup(&nativeMigrator)
 
 	srv := api.Rest{
-		Version:     revision,
-		DataService: dataService,
-		Exporter:    exporter,
-		Importer:    &migrator.Remark{CommentCreator: dataStore},
-		Disqus:      &migrator.Disqus{CommentCreator: dataStore},
-		WebRoot:     opts.ServerCommand.WebRoot,
+		Version:        revision,
+		DataService:    dataService,
+		NativeMigrator: nativeMigrator,
+		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
+		WebRoot:        opts.WebRoot,
 		Authenticator: auth.Authenticator{
 			Admins:       opts.Admins,
 			SessionStore: sessionStore,
@@ -132,7 +109,7 @@ func main() {
 		},
 		Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn),
 	}
-	srv.Run(opts.ServerCommand.Port)
+	srv.Run(opts.Port)
 }
 
 // activateBackup runs background backups for each site
@@ -203,19 +180,17 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
 		}
 	}
 
-	srvOpts := opts.ServerCommand
-
-	if srvOpts.GoogleCID != "" && srvOpts.GoogleCSEC != "" {
-		providers = append(providers, auth.NewGoogle(makeParams(srvOpts.GoogleCID, srvOpts.GoogleCSEC)))
+	if opts.GoogleCID != "" && opts.GoogleCSEC != "" {
+		providers = append(providers, auth.NewGoogle(makeParams(opts.GoogleCID, opts.GoogleCSEC)))
 	}
-	if srvOpts.GithubCID != "" && srvOpts.GithubCSEC != "" {
-		providers = append(providers, auth.NewGithub(makeParams(srvOpts.GithubCID, srvOpts.GithubCSEC)))
+	if opts.GithubCID != "" && opts.GithubCSEC != "" {
+		providers = append(providers, auth.NewGithub(makeParams(opts.GithubCID, opts.GithubCSEC)))
 	}
-	if srvOpts.FacebookCID != "" && srvOpts.FacebookCSEC != "" {
-		providers = append(providers, auth.NewFacebook(makeParams(srvOpts.FacebookCID, srvOpts.FacebookCSEC)))
+	if opts.FacebookCID != "" && opts.FacebookCSEC != "" {
+		providers = append(providers, auth.NewFacebook(makeParams(opts.FacebookCID, opts.FacebookCSEC)))
 	}
-	if srvOpts.DisqusCID != "" && srvOpts.DisqusCSEC != "" {
-		providers = append(providers, auth.NewDisqus(makeParams(srvOpts.DisqusCID, srvOpts.DisqusCSEC)))
+	if opts.DisqusCID != "" && opts.DisqusCSEC != "" {
+		providers = append(providers, auth.NewDisqus(makeParams(opts.DisqusCID, opts.DisqusCSEC)))
 	}
 	if len(providers) == 0 {
 		log.Printf("[WARN] no auth providers defined")
@@ -226,6 +201,7 @@ func makeAuthProviders(sessionStore sessions.Store, avatarProxy *auth.AvatarProx
 // post-flush callback invoked by cache after each flush in async way
 func postFlushFn() {
 
+	// list of heavy urls for pre-heating on cache change
 	urls := []string{
 		"http://localhost:%d/api/v1/list?site=%s",
 		"http://localhost:%d/api/v1/last/50?site=%s",
@@ -233,7 +209,7 @@ func postFlushFn() {
 
 	for _, site := range opts.Sites {
 		for _, u := range urls {
-			resp, err := http.Get(fmt.Sprintf(u, opts.ServerCommand.Port, site))
+			resp, err := http.Get(fmt.Sprintf(u, opts.Port, site))
 			if err != nil {
 				log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
 				return
diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go
index 8eada1e8..691f3f67 100644
--- a/app/rest/api/admin.go
+++ b/app/rest/api/admin.go
@@ -19,9 +19,8 @@ import (
 // admin provides router for all requests available for admin users only
 type admin struct {
 	dataService    store.Service
-	exporterNative migrator.Exporter
-	importerNative migrator.Importer
-	importerDisqus migrator.Importer
+	nativeMigrator migrator.Remark
+	disqusImporter migrator.Importer
 	cache          rest.LoadingCache
 	defAvatarURL   string
 }
@@ -108,7 +107,7 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		writer = gzip.NewWriter(w)
 	}
 
-	if err := a.exporterNative.Export(writer, siteID); err != nil {
+	if err := a.nativeMigrator.Export(writer, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
 	}
 }
@@ -117,13 +116,14 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
 // imports comments from post body.
 func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	importer := a.importerNative
+	var importer migrator.Importer = &a.nativeMigrator
 	if r.URL.Query().Get("provider") == "disqus" {
-		importer = a.importerDisqus
+		importer = a.disqusImporter
 	}
 
 	if err := importer.Import(r.Body, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
+		return
 	}
 	a.cache.Flush()
 }
diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go
index 7f39cbc0..da41c06a 100644
--- a/app/rest/api/admin_test.go
+++ b/app/rest/api/admin_test.go
@@ -160,3 +160,32 @@ func TestAdmin_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(body, "\"text\""))
 	t.Logf("%s", body)
 }
+
+func TestAdmin_Import(t *testing.T) {
+	srv, port := prep(t)
+	assert.NotNil(t, srv)
+	defer cleanup(srv)
+
+	// add 2 initial comments, will be deleted by import
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
+	c2 := store.Comment{Text: "test test #2",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
+	addComment(t, c1, port)
+	addComment(t, c2, port)
+
+	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
+		port), "application/json", r)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port))
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err = json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi)
+}
diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go
index bc7822d3..664877e5 100644
--- a/app/rest/api/rest.go
+++ b/app/rest/api/rest.go
@@ -35,9 +35,8 @@ type Rest struct {
 	Cache         rest.LoadingCache
 	WebRoot       string
 
-	Exporter migrator.Exporter
-	Importer migrator.Importer
-	Disqus   migrator.Importer
+	NativeMigrator migrator.Remark
+	DisqusImporter migrator.Importer
 
 	httpServer    *http.Server
 	amdminService admin
@@ -53,9 +52,8 @@ func (s *Rest) Run(port int) {
 
 	s.amdminService = admin{
 		dataService:    s.DataService,
-		exporterNative: s.Exporter,
-		importerNative: s.Importer,
-		importerDisqus: s.Disqus,
+		nativeMigrator: s.NativeMigrator,
+		disqusImporter: s.DisqusImporter,
 		cache:          s.Cache,
 		defAvatarURL:   s.Authenticator.AvatarProxy.Default(),
 	}
diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go
index fa110ddb..e7e34353 100644
--- a/app/rest/api/rest_test.go
+++ b/app/rest/api/rest_test.go
@@ -421,9 +421,10 @@ func prep(t *testing.T) (srv *Rest, port int) {
 			AvatarProxy:  &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"},
 			Admins:       []string{"a1", "a2"},
 		},
-		Exporter: &migrator.Remark{CommentFinder: dataStore},
-		Cache:    &mockCache{},
-		WebRoot:  "/tmp",
+		NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore},
+		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
+		Cache:          &mockCache{},
+		WebRoot:        "/tmp",
 	}
 
 	ioutil.WriteFile(testHTML, []byte("some html"), 0700)
diff --git a/remark.rest b/remark.rest
index a52a2126..206679a5 100644
--- a/remark.rest
+++ b/remark.rest
@@ -107,4 +107,3 @@ GET {{host}}/api/v1/avatar/blah
 
 ### ping
 GET {{host}}/ping
-
diff --git a/start.sh b/start.sh
index b8ee8fa0..3b87cb73 100755
--- a/start.sh
+++ b/start.sh
@@ -12,4 +12,4 @@ cp -nv /srv/web/remark.image /srv/var/avatars/remark.image
 
 echo "start remark42 server"
 
-/sbin/su-exec ${USER} /srv/remark server
+/sbin/su-exec ${USER} /srv/remark

From 4f8cabf69e0c5c688972613ee8e756d64a7583dc Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 30 Apr 2018 03:13:45 -0500
Subject: [PATCH 3/4] move import to local dedicated server

---
 README.md                   | 66 ++++++++++++++++++-------------------
 app/main.go                 | 26 ++++++++++-----
 app/rest/api/admin.go       | 29 ++++------------
 app/rest/api/admin_test.go  | 29 ----------------
 app/rest/api/import.go      | 59 +++++++++++++++++++++++++++++++++
 app/rest/api/import_test.go | 63 +++++++++++++++++++++++++++++++++++
 app/rest/api/rest.go        | 14 ++++----
 app/rest/api/rest_test.go   | 15 +++++++--
 import-disqus.sh            |  4 +++
 9 files changed, 200 insertions(+), 105 deletions(-)
 create mode 100644 app/rest/api/import.go
 create mode 100644 app/rest/api/import_test.go
 create mode 100644 import-disqus.sh

diff --git a/README.md b/README.md
index 964e6bc2..259328e3 100644
--- a/README.md
+++ b/README.md
@@ -23,34 +23,34 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 
 * copy provided `docker-compose.yml` and customize for your needs
 * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
-* prepare user id for container ``export USER=`id -u $USER` ``
+* prepare user id for container `` export USER=`id -u $USER` ``
 * make sure you **don't keep** `DEV=true` for any non-development deployments
 * pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from sources - `docker-compose build`
 
 #### Parameters
 
-| Command line    | Environment          | Default                | Multi | Scope  | Description                     |
-| --------------- | -------------------- | ---------------------- | ----- | ------ | ------------------------------- |
-| --url           | REMARK_URL           | `https://remark42.com` | no    | all    | url to remark server            |
-| --bolt          | BOLTDB_PATH          | `/tmp`                 | no    | all    | path to data directory          |
-| --site          | SITE                 | `remark`               | yes   | server | site name(s)                    |
-| --admin         | ADMIN                |                        | yes   | server | admin names (list of user ids)  |
-| --backup        | BACKUP_PATH          | `/tmp`                 | no    | server | backups location                |
-| --max-back      | MAX_BACKUP_FILES     | `10`                   | no    | server | max backup files to keep        |
-| --session       | SESSION_STORE        | `/tmp`                 | no    | server | path to session store directory |
-| --store-key     | STORE_KEY            | `secure-store-key`     | no    | server | session store encryption key    |
-| --google-cid    | REMARK_GOOGLE_CID    |                        | no    | server | Google OAuth client ID          |
-| --google-csec   | REMARK_GOOGLE_CSEC   |                        | no    | server | Google OAuth client secret      |
-| --facebook-cid  | REMARK_FACEBOOK_CID  |                        | no    | server | Facebook OAuth client ID        |
-| --facebook-csec | REMARK_FACEBOOK_CSEC |                        | no    | server | Facebook OAuth client secret    |
-| --github-cid    | REMARK_GITHUB_CID    |                        | no    | server | Github OAuth client ID          |
-| --github-csec   | REMARK_GITHUB_CSEC   |                        | no    | server | Github OAuth client secret      |
-| --provider      |                      | `disqus`               | no    | import | provider type for import        |
-| --site          |                      | `remark`               | no    | import | site ID                         |
-| --file          |                      | `disqus.xml`           | no    | import | import file                     |
-| --dbg           | DEBUG                | `false`                | no    | all    | debug mode                      |
-| --dev-password  | DEV_PASSWD           |                        | no    | all    | password for `dev` user         |
+| Command line    | Environment          | Default                | Multi | Description                     |
+| --------------- | -------------------- | ---------------------- | ----- | ------------------------------- |
+| --url           | REMARK_URL           | `https://remark42.com` | no    | url to remark server            |
+| --bolt          | BOLTDB_PATH          | `/tmp`                 | no    | path to data directory          |
+| --site          | SITE                 | `remark`               | yes   | site name(s)                    |
+| --admin         | ADMIN                |                        | yes   | admin names (list of user ids)  |
+| --backup        | BACKUP_PATH          | `/tmp`                 | no    | backups location                |
+| --max-back      | MAX_BACKUP_FILES     | `10`                   | no    | max backup files to keep        |
+| --session       | SESSION_STORE        | `/tmp`                 | no    | path to session store directory |
+| --store-key     | STORE_KEY            | `secure-store-key`     | no    | session store encryption key    |
+| --google-cid    | REMARK_GOOGLE_CID    |                        | no    | Google OAuth client ID          |
+| --google-csec   | REMARK_GOOGLE_CSEC   |                        | no    | Google OAuth client secret      |
+| --facebook-cid  | REMARK_FACEBOOK_CID  |                        | no    | Facebook OAuth client ID        |
+| --facebook-csec | REMARK_FACEBOOK_CSEC |                        | no    | Facebook OAuth client secret    |
+| --github-cid    | REMARK_GITHUB_CID    |                        | no    | Github OAuth client ID          |
+| --github-csec   | REMARK_GITHUB_CSEC   |                        | no    | Github OAuth client secret      |
+| --provider      |                      | `disqus`               | no    | provider type for import        |
+| --site          |                      | `remark`               | no    | site ID                         |
+| --file          |                      | `disqus.xml`           | no    | import file                     |
+| --dbg           | DEBUG                | `false`                | no    | debug mode                      |
+| --dev-password  | DEV_PASSWD           |                        | no    | password for `dev` user         |
 
 _all multi parameters separated by `,`_
 
@@ -101,10 +101,8 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
-2.  Move this file to your remark42 host and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Stop remark42 containers if started, i.e. `docker-compose stop`
-4.  Run import command - `docker-compose run --rm remark /srv/remark import --file=<disqus-export-name>.xml --site=<your site id>`
-5.  Start remark42 containers `docker-compose up -d`
+2.  Move this file to your remark42 host within `.var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
+3.  Run import command - `docker-compose run remark /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
 
 ### Frontend
 
@@ -112,20 +110,20 @@ Frontend part is building automatically along with backend if you use `docker-co
 
 For manual building:
 
-- install [Node.js 8](https://nodejs.org/en/) or higher;
-- run `npm install` inside `./web`;
-- run `npm run build` there;
-- result files will be saved in `./web/public`.
+* install [Node.js 8](https://nodejs.org/en/) or higher;
+* run `npm install` inside `./web`;
+* run `npm run build` there;
+* result files will be saved in `./web/public`.
 
 For development mode use `npm start` instead of `npm run build`.
 In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
 
 URLs for development:
 
-- `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
-- `localhost:8080/dev.html` — page with embedded script from local folder;
-- `localhost:8080/last-comments.html` — page with embedded script for last comments;
-- `localhost:8080/counter.html` — page with embedded script for counter with examples.
+* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
+* `localhost:8080/dev.html` — page with embedded script from local folder;
+* `localhost:8080/last-comments.html` — page with embedded script for last comments;
+* `localhost:8080/counter.html` — page with embedded script for counter with examples.
 
 ## API
 
diff --git a/app/main.go b/app/main.go
index 06b595df..9dce3b80 100644
--- a/app/main.go
+++ b/app/main.go
@@ -83,7 +83,18 @@ func main() {
 		return sess
 	}()
 
-	nativeMigrator := migrator.Remark{CommentFinder: dataStore}
+	exporter := migrator.Remark{CommentFinder: dataStore}
+	cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn)
+
+	activateBackup(&exporter)
+
+	importSrv := api.Import{
+		Version:        revision,
+		Cache:          cache,
+		NativeImporter: &migrator.Remark{CommentCreator: dataStore},
+		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
+	}
+	go importSrv.Run(opts.Port + 1)
 
 	avatarProxy := &auth.AvatarProxy{
 		StorePath:     opts.AvatarStore,
@@ -92,14 +103,11 @@ func main() {
 		DefaultAvatar: opts.DefaultAvatar,
 	}
 
-	activateBackup(&nativeMigrator)
-
 	srv := api.Rest{
-		Version:        revision,
-		DataService:    dataService,
-		NativeMigrator: nativeMigrator,
-		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
-		WebRoot:        opts.WebRoot,
+		Version:     revision,
+		DataService: dataService,
+		Exporter:    &exporter,
+		WebRoot:     opts.WebRoot,
 		Authenticator: auth.Authenticator{
 			Admins:       opts.Admins,
 			SessionStore: sessionStore,
@@ -107,7 +115,7 @@ func main() {
 			AvatarProxy:  avatarProxy,
 			DevPasswd:    opts.DevPasswd,
 		},
-		Cache: rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn),
+		Cache: cache,
 	}
 	srv.Run(opts.Port)
 }
diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go
index 691f3f67..b98eaf0a 100644
--- a/app/rest/api/admin.go
+++ b/app/rest/api/admin.go
@@ -18,11 +18,10 @@ import (
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService    store.Service
-	nativeMigrator migrator.Remark
-	disqusImporter migrator.Importer
-	cache          rest.LoadingCache
-	defAvatarURL   string
+	dataService  store.Service
+	exporter     migrator.Exporter
+	cache        rest.LoadingCache
+	defAvatarURL string
 }
 
 func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
@@ -31,7 +30,7 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
 	router.Delete("/comment/{id}", a.deleteCommentCtrl)
 	router.Put("/user/{userid}", a.setBlockCtrl)
 	router.Get("/export", a.exportCtrl)
-	router.Post("/import", a.importCtrl)
+
 	router.Put("/pin/{id}", a.setPinCtrl)
 	router.Get("/blocked", a.blockedUsersCtrl)
 	return router
@@ -107,27 +106,11 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		writer = gzip.NewWriter(w)
 	}
 
-	if err := a.nativeMigrator.Export(writer, siteID); err != nil {
+	if err := a.exporter.Export(writer, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
 	}
 }
 
-// POST /import?site=site-id&provider=disqus|remark
-// imports comments from post body.
-func (a *admin) importCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	var importer migrator.Importer = &a.nativeMigrator
-	if r.URL.Query().Get("provider") == "disqus" {
-		importer = a.disqusImporter
-	}
-
-	if err := importer.Import(r.Body, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
-		return
-	}
-	a.cache.Flush()
-}
-
 func (a *admin) checkBlocked(siteID string, user store.User) bool {
 	return a.dataService.IsBlocked(siteID, user.ID)
 }
diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go
index da41c06a..7f39cbc0 100644
--- a/app/rest/api/admin_test.go
+++ b/app/rest/api/admin_test.go
@@ -160,32 +160,3 @@ func TestAdmin_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(body, "\"text\""))
 	t.Logf("%s", body)
 }
-
-func TestAdmin_Import(t *testing.T) {
-	srv, port := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(srv)
-
-	// add 2 initial comments, will be deleted by import
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
-	c2 := store.Comment{Text: "test test #2",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blahX"}}
-	addComment(t, c1, port)
-	addComment(t, c2, port)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
-		port), "application/json", r)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-
-	body, code := get(t, fmt.Sprintf("http://127.0.0.1:%d/api/v1/list?site=radio-t", port))
-	assert.Equal(t, 200, code)
-	pi := []store.PostInfo{}
-	err = json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 1}, {URL: "https://radio-t.com/blah1", Count: 1}}, pi)
-}
diff --git a/app/rest/api/import.go b/app/rest/api/import.go
new file mode 100644
index 00000000..314c3fa7
--- /dev/null
+++ b/app/rest/api/import.go
@@ -0,0 +1,59 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/didip/tollbooth"
+	"github.com/didip/tollbooth_chi"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	"github.com/umputun/remark/app/migrator"
+	"github.com/umputun/remark/app/rest"
+)
+
+// Import rest runs on unexposed port and available for local requests only
+type Import struct {
+	Version        string
+	Cache          rest.LoadingCache
+	NativeImporter migrator.Importer
+	DisqusImporter migrator.Importer
+
+	httpServer *http.Server
+}
+
+// Run the listener and request's router, activate rest server
+// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
+func (s *Import) Run(port int) {
+	log.Printf("[INFO] activate import server on port %d", port)
+
+	router := chi.NewRouter()
+	router.Use(middleware.RealIP, Recoverer)
+	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+	router.Use(AppInfo("remark42-importer", s.Version), Ping, Logger(LogAll))
+
+	router.Post("/api/v1/admin/import", s.importCtrl)
+
+	s.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
+	err := s.httpServer.ListenAndServe()
+	log.Printf("[WARN] http server terminated, %s", err)
+}
+
+// POST /import?site=site-id&provider=disqus|remark
+// imports comments from post body.
+func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	importer := s.NativeImporter
+	if r.URL.Query().Get("provider") == "disqus" {
+		importer = s.DisqusImporter
+	}
+
+	if err := importer.Import(r.Body, siteID); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
+		return
+	}
+	s.Cache.Flush()
+}
diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go
new file mode 100644
index 00000000..f4045547
--- /dev/null
+++ b/app/rest/api/import_test.go
@@ -0,0 +1,63 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/app/migrator"
+	"github.com/umputun/remark/app/store"
+)
+
+func TestImport(t *testing.T) {
+	srv, port := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv)
+
+	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
+		port), "application/json", r)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func prepImportSrv(t *testing.T) (srv *Import, port int) {
+	dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.Nil(t, err)
+
+	srv = &Import{
+		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
+		NativeImporter: &migrator.Remark{CommentCreator: dataStore},
+		Cache:          &mockCache{},
+	}
+
+	portSetCh := make(chan bool)
+
+	go func() {
+		port = rand.Intn(50000) + 1025
+		portSetCh <- true
+		srv.Run(port)
+	}()
+
+	<-portSetCh
+
+	time.Sleep(100 * time.Millisecond)
+	return srv, port
+}
+
+func cleanupImportSrv(srv *Import) {
+	srv.httpServer.Close()
+	srv.httpServer.Shutdown(context.Background())
+	os.Remove(testDb)
+}
diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go
index 664877e5..67287a98 100644
--- a/app/rest/api/rest.go
+++ b/app/rest/api/rest.go
@@ -35,8 +35,7 @@ type Rest struct {
 	Cache         rest.LoadingCache
 	WebRoot       string
 
-	NativeMigrator migrator.Remark
-	DisqusImporter migrator.Importer
+	Exporter migrator.Exporter
 
 	httpServer    *http.Server
 	amdminService admin
@@ -44,18 +43,17 @@ type Rest struct {
 
 // Run the lister and request's router, activate rest server
 func (s *Rest) Run(port int) {
-	log.Print("[INFO] activate rest server")
+	log.Printf("[INFO] activate rest server on port %d", port)
 
 	if len(s.Authenticator.Admins) > 0 {
 		log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
 	}
 
 	s.amdminService = admin{
-		dataService:    s.DataService,
-		nativeMigrator: s.NativeMigrator,
-		disqusImporter: s.DisqusImporter,
-		cache:          s.Cache,
-		defAvatarURL:   s.Authenticator.AvatarProxy.Default(),
+		dataService:  s.DataService,
+		exporter:     s.Exporter,
+		cache:        s.Cache,
+		defAvatarURL: s.Authenticator.AvatarProxy.Default(),
 	}
 
 	router := chi.NewRouter()
diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go
index e7e34353..714b6c9f 100644
--- a/app/rest/api/rest_test.go
+++ b/app/rest/api/rest_test.go
@@ -421,21 +421,32 @@ func prep(t *testing.T) (srv *Rest, port int) {
 			AvatarProxy:  &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"},
 			Admins:       []string{"a1", "a2"},
 		},
-		NativeMigrator: migrator.Remark{CommentFinder: dataStore, CommentCreator: dataStore},
+		Exporter: &migrator.Remark{CommentFinder: dataStore},
+		Cache:    &mockCache{},
+		WebRoot:  "/tmp",
+	}
+
+	importSrv := &Import{
 		DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
+		NativeImporter: &migrator.Remark{CommentCreator: dataStore},
 		Cache:          &mockCache{},
-		WebRoot:        "/tmp",
 	}
 
 	ioutil.WriteFile(testHTML, []byte("some html"), 0700)
 	portSetCh := make(chan bool)
+
 	go func() {
 		port = rand.Intn(50000) + 1025
 		portSetCh <- true
 		srv.Run(port)
 	}()
+
 	<-portSetCh
 
+	go func() {
+		importSrv.Run(port + 1)
+	}()
+
 	time.Sleep(100 * time.Millisecond)
 	return srv, port
 }
diff --git a/import-disqus.sh b/import-disqus.sh
new file mode 100644
index 00000000..cba716dc
--- /dev/null
+++ b/import-disqus.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+echo "import disqus file $1 to site $2"
+curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark &
+provider=disqus

From 77ded39de2cfa67e3398d826b0b6db707a8b97f8 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Tue, 1 May 2018 20:53:40 -0500
Subject: [PATCH 4/4] fix readme ref to import mode

---
 README.md | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/README.md b/README.md
index 259328e3..2e2600f4 100644
--- a/README.md
+++ b/README.md
@@ -54,11 +54,6 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 
 _all multi parameters separated by `,`_
 
-#### Run modes
-
-* `server` activates regular, server mode
-* `import` performs import from external providers (disqus and internal json, see `/api/v1/admin/export`)
-
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them in order to allow users to make comments. It is not mandatory to have all of them, but at least one should be property configured.