diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go
index fe63910f..3d4da5e8 100644
--- a/app/migrator/disqus.go
+++ b/app/migrator/disqus.go
@@ -54,17 +54,20 @@ type uid struct {
func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
commentsCh := d.convert(r, siteID)
- failed := 0
+ failed, passed := 0, 0
for c := range commentsCh {
if _, err = d.DataStore.Create(c); err != nil {
failed++
+ continue
}
+ passed++
}
if failed > 0 {
return errors.Errorf("failed to save %d comments", failed)
}
+ log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
return nil
}
@@ -74,8 +77,8 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
decoder := xml.NewDecoder(r)
commentsCh := make(chan store.Comment)
- inpThreads, inpComments := 0, 0
go func() {
+ inpThreads, inpComments := 0, 0
commentsCount, spamComments := 0, 0
for {
t, err := decoder.Token()
@@ -97,6 +100,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
inpComments++
comment := disqusComment{}
if err := decoder.DecodeElement(&comment, &se); err != nil {
+ log.Printf("[WARN] can't decode disqus comment, %s", err)
continue
}
if comment.IsSpam {
@@ -127,7 +131,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
}
}
close(commentsCh)
- log.Printf("[INFO] converted %d posts with %d comments from disqus %d/%d, spam %d",
+ log.Printf("[INFO] converted %d posts with %d comments from disqus (threads:%d, comments:%d, spam:%d)",
len(postsMap), commentsCount, inpThreads, inpComments, spamComments)
}()
diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go
index b31ebf04..85fbe706 100644
--- a/app/migrator/disqus_test.go
+++ b/app/migrator/disqus_test.go
@@ -1,14 +1,38 @@
package migrator
import (
+ "os"
"strings"
"testing"
"time"
+ "github.com/stretchr/testify/require"
+
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/app/store"
)
+func TestDisqus_Import(t *testing.T) {
+ defer os.Remove("/tmp/remark-test.db")
+ dataStore, err := store.NewBoltDB(store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+ require.Nil(t, err, "create store")
+ d := Disqus{DataStore: dataStore}
+ err = d.Import(strings.NewReader(xmlTest), "test")
+ assert.Nil(t, err)
+
+ last, err := dataStore.Last("test", 10)
+ assert.Nil(t, err)
+ assert.Equal(t, 3, len(last), "3 comments imported")
+
+ posts, err := dataStore.List("test")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(posts), "2 posts")
+
+ count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
+ assert.Nil(t, err)
+ assert.Equal(t, 2, count)
+}
+
func TestDisqus_Convert(t *testing.T) {
d := Disqus{}
ch := d.convert(strings.NewReader(xmlTest), "test")
@@ -16,7 +40,6 @@ func TestDisqus_Convert(t *testing.T) {
res := []store.Comment{}
for comment := range ch {
res = append(res, comment)
- t.Logf("%+v", comment)
}
assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded")
@@ -134,15 +157,13 @@ var xmlTest = `
google-74b9e7568ef6860e93862c5d77590123
89.89.89.139
-
+
6580890074280459219
-
- some ugly spam
-
- 2011-09-31T22:48:43Z
+ some ugly spam
+ 2011-09-30T22:48:43Z
false
true
diff --git a/app/store/bolt.go b/app/store/bolt.go
index fcab2b66..5a76c604 100644
--- a/app/store/bolt.go
+++ b/app/store/bolt.go
@@ -258,7 +258,7 @@ func (b *BoltDB) Count(locator Locator) (count int, err error) {
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(locator.URL))
if bucket == nil {
- return errors.Errorf("no bucket %s in store", locator.URL)
+ return errors.Errorf("no bucket %s in store %s", locator.URL, locator.SiteID)
}
count = bucket.Stats().KeyN
return nil