mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

return import status

Browse Source
This commit is contained in:
Umputun
2018-05-02 02:22:48 -05:00
parent e1fd9bb7a8
commit a8033bfeb1
12 changed files with 44 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/migrator/backup.go
+1 -1
View File
@@ -46,7 +46,7 @@ func (ab AutoBackup) makeBackup() (string, error) {
}
gz := gzip.NewWriter(fh)
if err = ab.Exporter.Export(gz, ab.SiteID); err != nil {
if _, err = ab.Exporter.Export(gz, ab.SiteID); err != nil {
return "", errors.Wrapf(err, "export failed for %s", ab.SiteID)
}
if err = gz.Close(); err != nil {
app/migrator/backup_test.go
+2 -2
View File
@@ -54,7 +54,7 @@ func TestMigrator_MakeBackup(t *testing.T) {
type mockExporter struct{}
func (mock *mockExporter) Export(w io.Writer, siteID string) error {
func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
w.Write([]byte("some export blah blah 1234567890"))
return nil
return 1000, nil
}
app/migrator/disqus.go
+4 -4
View File
@@ -51,10 +51,10 @@ type uid struct {
}
// Import from disqus and save to store
func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
if err = d.DeleteAll(siteID); err != nil {
return err
return 0, err
}
commentsCh := d.convert(r, siteID)
@@ -68,7 +68,7 @@ func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
}
if failed > 0 {
return errors.Errorf("failed to save %d comments", failed)
return passed, errors.Errorf("failed to save %d comments", failed)
}
log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
@@ -76,7 +76,7 @@ func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
if failed > 0 && passed == 0 {
err = errors.New("import failed")
}
return err
return passed, err
}
// convert disqus stream (xml) from reader and fill channel of comments.
app/migrator/disqus_test.go
+2 -1
View File
@@ -18,8 +18,9 @@ func TestDisqus_Import(t *testing.T) {
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
d := Disqus{CommentCreator: dataStore}
err = d.Import(strings.NewReader(xmlTest), "test")
size, err := d.Import(strings.NewReader(xmlTest), "test")
assert.Nil(t, err)
assert.Equal(t, 3, size)
last, err := dataStore.Last("test", 10)
assert.Nil(t, err)
app/migrator/migrator.go
+5 -5
View File
@@ -15,12 +15,12 @@ import (
// Importer defines interface to convert posts from external sources
type Importer interface {
Import(r io.Reader, siteID string) error
Import(r io.Reader, siteID string) (int, error)
}
// Exporter defines interface to export comments from internal store
type Exporter interface {
Export(w io.Writer, siteID string) error
Export(w io.Writer, siteID string) (int, error)
}
// CommentCreator is a minimal interface used by importer to make comments
@@ -44,7 +44,7 @@ type ImportParams struct {
}
// ImportComments imports from given provider format and saves to store
func ImportComments(p ImportParams) error {
func ImportComments(p ImportParams) (int, error) {
log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)
var importer Importer
@@ -54,12 +54,12 @@ func ImportComments(p ImportParams) error {
case "native":
importer = &Remark{CommentCreator: p.CommentCreator}
default:
return errors.Errorf("unsupported import provider %s", p.Provider)
return 0, errors.Errorf("unsupported import provider %s", p.Provider)
}
fh, err := os.Open(p.InputFile)
if err != nil {
return errors.Wrapf(err, "can't open import file %s", p.InputFile)
return 0, errors.Wrapf(err, "can't open import file %s", p.InputFile)
}
defer func() {
app/migrator/migrator_test.go
+4 -2
View File
@@ -24,13 +24,14 @@ func TestMigrator_ImportDisqus(t *testing.T) {
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
err = ImportComments(ImportParams{
size, err := ImportComments(ImportParams{
CommentCreator: dataStore,
InputFile: "/tmp/disqus-test.xml",
SiteID: "test",
Provider: "disqus",
})
assert.Nil(t, err)
assert.Equal(t, 3, size)
last, err := dataStore.Last("test", 10)
assert.Nil(t, err)
@@ -52,13 +53,14 @@ func TestMigrator_ImportRemark(t *testing.T) {
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
require.Nil(t, err, "create store")
err = ImportComments(ImportParams{
size, err := ImportComments(ImportParams{
CommentCreator: dataStore,
InputFile: "/tmp/disqus-test.r42",
SiteID: "radio-t",
Provider: "native",
})
assert.Nil(t, err)
assert.Equal(t, 2, size)
last, err := dataStore.Last("radio-t", 10)
assert.Nil(t, err)
app/migrator/remark.go
+11 -11
View File
@@ -19,10 +19,10 @@ type Remark struct {
}
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
func (r *Remark) Export(w io.Writer, siteID string) error {
func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
topics, err := r.List(siteID, 0, 0)
if err != nil {
return err
return 0, err
}
log.Printf("[DEBUG] exporting %d topics", len(topics))
@@ -31,7 +31,7 @@ func (r *Remark) Export(w io.Writer, siteID string) error {
topic := topics[i]
comments, err := r.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
if err != nil {
return err
return commentsCount, err
}
for _, comment := range comments {
@@ -41,24 +41,24 @@ func (r *Remark) Export(w io.Writer, siteID string) error {
enc.SetEscapeHTML(false)
if err := enc.Encode(comment); err != nil {
return errors.Wrapf(err, "can't marshal %v", comments)
return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
}
data := buf.Bytes()
if _, err := w.Write(data); err != nil {
return errors.Wrap(err, "can't write comment data")
return commentsCount, errors.Wrap(err, "can't write comment data")
}
commentsCount++
}
}
log.Printf("[DEBUG] exported %d comments", commentsCount)
return nil
return commentsCount, nil
}
// Import comments from json strings produced by Remark.Export
func (r *Remark) Import(reader io.Reader, siteID string) error {
func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
if err := r.DeleteAll(siteID); err != nil {
return err
return 0, err
}
failed := 0
@@ -87,11 +87,11 @@ func (r *Remark) Import(reader io.Reader, siteID string) error {
}
}
if scanner.Err() != nil {
return errors.Wrap(scanner.Err(), "error in scan")
return comments, errors.Wrap(scanner.Err(), "error in scan")
}
if failed > 0 {
return errors.Errorf("failed to save %d comments", failed)
return comments, errors.Errorf("failed to save %d comments", failed)
}
log.Printf("[INFO] imported %d comments from %d records", comments, total)
return nil
return comments, nil
}
app/migrator/remark_test.go
+4 -2
View File
@@ -20,8 +20,9 @@ func TestRemark_Export(t *testing.T) {
r := Remark{CommentFinder: b}
buf := &bytes.Buffer{}
err := r.Export(buf, "radio-t")
size, err := r.Export(buf, "radio-t")
assert.Nil(t, err)
assert.Equal(t, 2, size)
c1, err := buf.ReadString('\n')
assert.Nil(t, err)
@@ -43,8 +44,9 @@ func TestRemark_Import(t *testing.T) {
b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{CommentCreator: b}
err = r.Import(buf, "radio-t")
size, err := r.Import(buf, "radio-t")
assert.Nil(t, err)
assert.Equal(t, 2, size)
comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
assert.Nil(t, err)
app/rest/api/admin.go
+3 -1
View File
@@ -106,9 +106,11 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
writer = gzip.NewWriter(w)
}
if err := a.exporter.Export(writer, siteID); err != nil {
if _, err := a.exporter.Export(writer, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
}
func (a *admin) checkBlocked(siteID string, user store.User) bool {
app/rest/api/import.go
+6 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/didip/tollbooth_chi"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/render"
"github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest"
)
@@ -51,9 +52,13 @@ func (s *Import) importCtrl(w http.ResponseWriter, r *http.Request) {
importer = s.DisqusImporter
}
if err := importer.Import(r.Body, siteID); err != nil {
size, err := importer.Import(r.Body, siteID)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
return
}
s.Cache.Flush()
render.Status(r, http.StatusCreated)
render.JSON(w, r, JSON{"status": "ok", "size": size})
}
app/rest/api/import_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ func TestImport(t *testing.T) {
resp, err := http.Post(fmt.Sprintf("http://dev:password@127.0.0.1:%d/api/v1/admin/import?site=radio-t&provider=native",
port), "application/json", r)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
}
func prepImportSrv(t *testing.T) (srv *Import, port int) {
import-disqus.sh
+1 -2
View File
@@ -1,4 +1,3 @@
#!/bin/sh
echo "import disqus file $1 to site $2"
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 http://127.0.0.1:8081/api/v1/admin/import?site=remark &
provider=disqus
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus"