diff --git a/app/migrator/backup.go b/app/migrator/backup.go index 8daf24b3..155ef75c 100644 --- a/app/migrator/backup.go +++ b/app/migrator/backup.go @@ -46,7 +46,7 @@ func (ab AutoBackup) makeBackup() (string, error) { } gz := gzip.NewWriter(fh) - if err = ab.Exporter.Export(gz, ab.SiteID); err != nil { + if _, err = ab.Exporter.Export(gz, ab.SiteID); err != nil { return "", errors.Wrapf(err, "export failed for %s", ab.SiteID) } if err = gz.Close(); err != nil { diff --git a/app/migrator/backup_test.go b/app/migrator/backup_test.go index cb28122a..a8e11779 100644 --- a/app/migrator/backup_test.go +++ b/app/migrator/backup_test.go @@ -54,7 +54,7 @@ func TestMigrator_MakeBackup(t *testing.T) { type mockExporter struct{} -func (mock *mockExporter) Export(w io.Writer, siteID string) error { +func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) { w.Write([]byte("some export blah blah 1234567890")) - return nil + return 1000, nil } diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index fd840dd0..d030c59a 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -51,10 +51,10 @@ type uid struct { } // Import from disqus and save to store -func (d *Disqus) Import(r io.Reader, siteID string) (err error) { +func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) { if err = d.DeleteAll(siteID); err != nil { - return err + return 0, err } commentsCh := d.convert(r, siteID) @@ -68,7 +68,7 @@ func (d *Disqus) Import(r io.Reader, siteID string) (err error) { } if failed > 0 { - return errors.Errorf("failed to save %d comments", failed) + return passed, errors.Errorf("failed to save %d comments", failed) } log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID) @@ -76,7 +76,7 @@ func (d *Disqus) Import(r io.Reader, siteID string) (err error) { if failed > 0 && passed == 0 { err = errors.New("import failed") } - return err + return passed, err } // convert disqus stream (xml) from reader and fill channel of comments. diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index 1c5740a6..b71c7f60 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -18,8 +18,9 @@ func TestDisqus_Import(t *testing.T) { dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") d := Disqus{CommentCreator: dataStore} - err = d.Import(strings.NewReader(xmlTest), "test") + size, err := d.Import(strings.NewReader(xmlTest), "test") assert.Nil(t, err) + assert.Equal(t, 3, size) last, err := dataStore.Last("test", 10) assert.Nil(t, err) diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go index 41bf93aa..b1912b7d 100644 --- a/app/migrator/migrator.go +++ b/app/migrator/migrator.go @@ -15,12 +15,12 @@ import ( // Importer defines interface to convert posts from external sources type Importer interface { - Import(r io.Reader, siteID string) error + Import(r io.Reader, siteID string) (int, error) } // Exporter defines interface to export comments from internal store type Exporter interface { - Export(w io.Writer, siteID string) error + Export(w io.Writer, siteID string) (int, error) } // CommentCreator is a minimal interface used by importer to make comments @@ -44,7 +44,7 @@ type ImportParams struct { } // ImportComments imports from given provider format and saves to store -func ImportComments(p ImportParams) error { +func ImportComments(p ImportParams) (int, error) { log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID) var importer Importer @@ -54,12 +54,12 @@ func ImportComments(p ImportParams) error { case "native": importer = &Remark{CommentCreator: p.CommentCreator} default: - return errors.Errorf("unsupported import provider %s", p.Provider) + return 0, errors.Errorf("unsupported import provider %s", p.Provider) } fh, err := os.Open(p.InputFile) if err != nil { - return errors.Wrapf(err, "can't open import file %s", p.InputFile) + return 0, errors.Wrapf(err, "can't open import file %s", p.InputFile) } defer func() { diff --git a/app/migrator/migrator_test.go b/app/migrator/migrator_test.go index 203205da..6a38ea1b 100644 --- a/app/migrator/migrator_test.go +++ b/app/migrator/migrator_test.go @@ -24,13 +24,14 @@ func TestMigrator_ImportDisqus(t *testing.T) { dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - err = ImportComments(ImportParams{ + size, err := ImportComments(ImportParams{ CommentCreator: dataStore, InputFile: "/tmp/disqus-test.xml", SiteID: "test", Provider: "disqus", }) assert.Nil(t, err) + assert.Equal(t, 3, size) last, err := dataStore.Last("test", 10) assert.Nil(t, err) @@ -52,13 +53,14 @@ func TestMigrator_ImportRemark(t *testing.T) { dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") - err = ImportComments(ImportParams{ + size, err := ImportComments(ImportParams{ CommentCreator: dataStore, InputFile: "/tmp/disqus-test.r42", SiteID: "radio-t", Provider: "native", }) assert.Nil(t, err) + assert.Equal(t, 2, size) last, err := dataStore.Last("radio-t", 10) assert.Nil(t, err) diff --git a/app/migrator/remark.go b/app/migrator/remark.go index f92aca04..823778db 100644 --- a/app/migrator/remark.go +++ b/app/migrator/remark.go @@ -19,10 +19,10 @@ type Remark struct { } // Export all comments to writer as json strings. Each comment is one string, separated by "\n" -func (r *Remark) Export(w io.Writer, siteID string) error { +func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) { topics, err := r.List(siteID, 0, 0) if err != nil { - return err + return 0, err } log.Printf("[DEBUG] exporting %d topics", len(topics)) @@ -31,7 +31,7 @@ func (r *Remark) Export(w io.Writer, siteID string) error { topic := topics[i] comments, err := r.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") if err != nil { - return err + return commentsCount, err } for _, comment := range comments { @@ -41,24 +41,24 @@ func (r *Remark) Export(w io.Writer, siteID string) error { enc.SetEscapeHTML(false) if err := enc.Encode(comment); err != nil { - return errors.Wrapf(err, "can't marshal %v", comments) + return commentsCount, errors.Wrapf(err, "can't marshal %v", comments) } data := buf.Bytes() if _, err := w.Write(data); err != nil { - return errors.Wrap(err, "can't write comment data") + return commentsCount, errors.Wrap(err, "can't write comment data") } commentsCount++ } } log.Printf("[DEBUG] exported %d comments", commentsCount) - return nil + return commentsCount, nil } // Import comments from json strings produced by Remark.Export -func (r *Remark) Import(reader io.Reader, siteID string) error { +func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) { if err := r.DeleteAll(siteID); err != nil { - return err + return 0, err } failed := 0 @@ -87,11 +87,11 @@ func (r *Remark) Import(reader io.Reader, siteID string) error { } } if scanner.Err() != nil { - return errors.Wrap(scanner.Err(), "error in scan") + return comments, errors.Wrap(scanner.Err(), "error in scan") } if failed > 0 { - return errors.Errorf("failed to save %d comments", failed) + return comments, errors.Errorf("failed to save %d comments", failed) } log.Printf("[INFO] imported %d comments from %d records", comments, total) - return nil + return comments, nil } diff --git a/app/migrator/remark_test.go b/app/migrator/remark_test.go index 1302e195..cc50c4fb 100644 --- a/app/migrator/remark_test.go +++ b/app/migrator/remark_test.go @@ -20,8 +20,9 @@ func TestRemark_Export(t *testing.T) { r := Remark{CommentFinder: b} buf := &bytes.Buffer{} - err := r.Export(buf, "radio-t") + size, err := r.Export(buf, "radio-t") assert.Nil(t, err) + assert.Equal(t, 2, size) c1, err := buf.ReadString('\n') assert.Nil(t, err) @@ -43,8 +44,9 @@ func TestRemark_Import(t *testing.T) { b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) r := Remark{CommentCreator: b} - err = r.Import(buf, "radio-t") + size, err := r.Import(buf, "radio-t") assert.Nil(t, err) + assert.Equal(t, 2, size) comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time") assert.Nil(t, err) diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index b98eaf0a..3ed95a8f 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -106,9 +106,11 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { writer = gzip.NewWriter(w) } - if err := a.exporter.Export(writer, siteID); err != nil { + if _, err := a.exporter.Export(writer, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") + return } + } func (a *admin) checkBlocked(siteID string, user store.User) bool { diff --git a/app/rest/api/import.go b/app/rest/api/import.go index 314c3fa7..883b64f5 100644 --- a/app/rest/api/import.go +++ b/app/rest/api/import.go @@ -10,6 +10,7 @@ import ( "github.com/didip/tollbooth_chi" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" + "github.com/go-chi/render" "github.com/umputun/remark/app/migrator" "github.com/umputun/remark/app/rest" ) @@ -51,9 +52,13 @@ func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) { importer = s.DisqusImporter } - if err := importer.Import(r.Body, siteID); err != nil { + size, err := importer.Import(r.Body, siteID) + if err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed") return } s.Cache.Flush() + + render.Status(r, http.StatusCreated) + render.JSON(w, r, JSON{"status": "ok", "size": size}) } diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go index f4045547..2d7f4e8b 100644 --- a/app/rest/api/import_test.go +++ b/app/rest/api/import_test.go @@ -29,7 +29,7 @@ func TestImport(t *testing.T) { resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native", port), "application/json", r) assert.Nil(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, http.StatusCreated, resp.StatusCode) } func prepImportSrv(t *testing.T) (srv *Import, port int) { diff --git a/import-disqus.sh b/import-disqus.sh index cba716dc..e79164af 100644 --- a/import-disqus.sh +++ b/import-disqus.sh @@ -1,4 +1,3 @@ #!/bin/sh echo "import disqus file $1 to site $2" -curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark & -provider=disqus +curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus"