add import test

app/migrator/disqus.go

@@ -54,17 +54,20 @@ type uid struct {
 func (d *Disqus) Import(r io.Reader, siteID string) (err error) {
 
 	commentsCh := d.convert(r, siteID)
-	failed := 0
+	failed, passed := 0, 0
 	for c := range commentsCh {
 		if _, err = d.DataStore.Create(c); err != nil {
 			failed++
+			continue
 		}
+		passed++
 	}
 
 	if failed > 0 {
 		return errors.Errorf("failed to save %d comments", failed)
 	}
 
+	log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
 	return nil
 }
 
@@ -74,8 +77,8 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 	decoder := xml.NewDecoder(r)
 	commentsCh := make(chan store.Comment)
 
-	inpThreads, inpComments := 0, 0
 	go func() {
+		inpThreads, inpComments := 0, 0
 		commentsCount, spamComments := 0, 0
 		for {
 			t, err := decoder.Token()
@@ -97,6 +100,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 					inpComments++
 					comment := disqusComment{}
 					if err := decoder.DecodeElement(&comment, &se); err != nil {
+						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						continue
 					}
 					if comment.IsSpam {
@@ -127,7 +131,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 			}
 		}
 		close(commentsCh)
-		log.Printf("[INFO] converted %d posts with %d comments from disqus %d/%d, spam %d",
+		log.Printf("[INFO] converted %d posts with %d comments from disqus (threads:%d, comments:%d, spam:%d)",
 			len(postsMap), commentsCount, inpThreads, inpComments, spamComments)
 	}()
 

app/migrator/disqus_test.go

@@ -1,14 +1,38 @@
 package migrator
 
 import (
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/umputun/remark/app/store"
 )
 
+func TestDisqus_Import(t *testing.T) {
+	defer os.Remove("/tmp/remark-test.db")
+	dataStore, err := store.NewBoltDB(store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.Nil(t, err, "create store")
+	d := Disqus{DataStore: dataStore}
+	err = d.Import(strings.NewReader(xmlTest), "test")
+	assert.Nil(t, err)
+
+	last, err := dataStore.Last("test", 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(last), "3 comments imported")
+
+	posts, err := dataStore.List("test")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(posts), "2 posts")
+
+	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
+	assert.Nil(t, err)
+	assert.Equal(t, 2, count)
+}
+
 func TestDisqus_Convert(t *testing.T) {
 	d := Disqus{}
 	ch := d.convert(strings.NewReader(xmlTest), "test")
@@ -16,7 +40,6 @@ func TestDisqus_Convert(t *testing.T) {
 	res := []store.Comment{}
 	for comment := range ch {
 		res = append(res, comment)
-		t.Logf("%+v", comment)
 	}
 	assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded")
 
@@ -134,15 +157,13 @@ var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 			<username>google-74b9e7568ef6860e93862c5d77590123</username>
 		</author>
 		<ipAddress>89.89.89.139</ipAddress>
-		<thread dsq:id="247937687"/>
+		<thread dsq:id="247918464"/>
 	</post>
 
 	<post dsq:id="299986073">
 		<id>6580890074280459219</id>
-		<message>
-			some ugly spam
-		</message>
-		<createdAt>2011-09-31T22:48:43Z</createdAt>
+		<message>some ugly spam</message>
+		<createdAt>2011-09-30T22:48:43Z</createdAt>
 		<isDeleted>false</isDeleted>
 		<isSpam>true</isSpam>
 		<author>

app/store/bolt.go

@@ -258,7 +258,7 @@ func (b *BoltDB) Count(locator Locator) (count int, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(locator.URL))
 		if bucket == nil {
-			return errors.Errorf("no bucket %s in store", locator.URL)
+			return errors.Errorf("no bucket %s in store %s", locator.URL, locator.SiteID)
 		}
 		count = bucket.Stats().KeyN
 		return nil