add export support

README.md

@@ -63,3 +63,4 @@ type Locator struct {
 - `PUT /api/v1/vote/{id}?url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decreases. _auth required_
 - `DELETE /api/v1/moderate/comment/{id}?url=post-url` - delete comment by `id`. _auth and admin required_
 - `PUT /api/v1/moderate/user/{userid}?site=side-id&block=1` - block or unblock user. _auth and admin required_
+- `GET /api/v1/export?site=side-id&block=1` - export all comments. _auth and admin required_

app/main.go

@@ -72,6 +72,7 @@ func main() {
 		SessionStore: sessionStore,
 		Admins:       opts.Admins,
 		DevMode:      opts.DevMode,
+		Exporter:     &migrator.Remark{DataStore: dataStore},
 		AuthGoogle: auth.NewGoogle(auth.Params{
 			Cid:          opts.ServerCommand.GoogleCID,
 			Csecret:      opts.ServerCommand.GoogleCSEC,

app/migrator/remark.go

@@ -0,0 +1,52 @@
+package migrator
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+
+	"github.com/pkg/errors"
+	"github.com/umputun/remark/app/store"
+)
+
+// Remark implements exporter and importer for internal store
+type Remark struct {
+	DataStore store.Interface
+}
+
+// Export all comments to writer as json
+func (r *Remark) Export(w io.Writer, siteID string) error {
+	topics, err := r.DataStore.List(store.Locator{SiteID: siteID})
+	if err != nil {
+		return err
+	}
+	log.Printf("[DEBUG] exporting %d topics", len(topics))
+
+	commentsCount := 0
+	for _, topic := range topics {
+		comments, err := r.DataStore.Find(store.Request{Locator: store.Locator{SiteID: siteID, URL: topic}})
+		if err != nil {
+			return err
+		}
+
+		for _, comment := range comments {
+
+			buf := &bytes.Buffer{}
+			enc := json.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+
+			if err := enc.Encode(comment); err != nil {
+				return errors.Wrapf(err, "can't marshal %v", comments)
+			}
+			data := buf.Bytes()
+			data = append(data, '\n')
+			if _, err := w.Write(data); err != nil {
+				return errors.Wrap(err, "can't write comment data")
+			}
+			commentsCount++
+		}
+	}
+	log.Printf("[DEBUG] exported %d comments", commentsCount)
+	return nil
+}

app/migrator/remark_test.go

@@ -0,0 +1,57 @@
+package migrator
+
+import (
+	"bytes"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/umputun/remark/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestRemark_Export(t *testing.T) {
+	b := prep(t)
+	r := Remark{DataStore: b}
+
+	buf := &bytes.Buffer{}
+	err := r.Export(buf, "radio-t")
+	assert.Nil(t, err)
+
+	c1, err := buf.ReadString('\n')
+	assert.Nil(t, err)
+	log.Print(c1)
+	exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
+	assert.Equal(t, exp, c1)
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) *store.BoltDB {
+	os.Remove(testDb)
+
+	b, err := store.NewBoltDB(testDb)
+	assert.Nil(t, err)
+
+	comment := store.Comment{
+		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	comment = store.Comment{
+		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	return b
+}

app/rest/server.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"encoding/json"
 
+	"github.com/umputun/remark/app/migrator"
 	"github.com/umputun/remark/app/rest/auth"
 	"github.com/umputun/remark/app/store"
 )
@@ -29,6 +30,7 @@ type Server struct {
 	AuthGoogle   *auth.Provider
 	AuthGithub   *auth.Provider
 	SessionStore *sessions.FilesystemStore
+	Exporter     migrator.Exporter
 	DevMode      bool
 
 	mod moderator
@@ -60,9 +62,11 @@ func (s *Server) Run() {
 			rauth.Put("/vote/{id}", s.voteCtrl)
 		})
 
-		rapi.With(Auth(s.SessionStore, s.Admins, s.DevMode)).Group(func(rmoder chi.Router) {
+		rapi.With(Auth(s.SessionStore, s.Admins, s.DevMode)).Group(func(radmin chi.Router) {
 			s.mod = moderator{dataStore: s.Store}
-			rmoder.Mount("/moderate", s.mod.routes())
+			radmin.Get("/export", s.exportCtrl)
+			radmin.Mount("/moderate", s.mod.routes())
+
 		})
 	})
 
@@ -249,6 +253,12 @@ func (s *Server) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, comment)
 }
 
+// GET /export?site=site-id
+func (s *Server) exportCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	s.Exporter.Export(w, siteID)
+}
+
 func httpError(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
 	render.Status(r, code)
 	render.JSON(w, r, JSON{"error": err.Error(), "details": details})

app/store/bolt.go

@@ -325,16 +325,18 @@ func (b *BoltDB) bucketForBlock(locator Locator, userID string) []byte {
 	return []byte(fmt.Sprintf("%s%s", blocksBucketPrefix, locator.SiteID))
 }
 
-// buckets returns list of buckets, which is list of all commented posts
-func (b BoltDB) buckets() (result []string) {
+// List returns list of buckets, which is list of all commented posts
+func (b BoltDB) List(locator Locator) (result []string, err error) {
 
-	_ = b.View(func(tx *bolt.Tx) error {
+	err = b.View(func(tx *bolt.Tx) error {
 		return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
-			result = append(result, string(name))
+			if string(name) != lastBucketName {
+				result = append(result, string(name))
+			}
 			return nil
 		})
 	})
-	return result
+	return result, err
 }
 
 type ref struct {

app/store/bolt_test.go

@@ -18,7 +18,7 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
 	res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}})
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>", res[0].Text)
+	assert.Equal(t, `some text, <a href="http://radio-t.com" rel="nofollow">link</a>`, res[0].Text)
 	assert.Equal(t, "user1", res[0].User.ID)
 	t.Log(res[0].ID)
 }
@@ -126,6 +126,21 @@ func TestBoltDB_BlockUser(t *testing.T) {
 
 }
 
+func TestBoltDB_List(t *testing.T) {
+	defer os.Remove(testDb)
+	b := prep(t) // two comments for https://radio-t.com
+
+	// add one more for https://radio-t.com/2
+	comment := Comment{Text: `some text, <a href="http://radio-t.com">link</a>`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator: Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
+	_, err := b.Create(comment)
+	assert.Nil(t, err)
+
+	res, err := b.List(Locator{SiteID: "site1"})
+	assert.Nil(t, err)
+	assert.Equal(t, []string{"https://radio-t.com", "https://radio-t.com/2"}, res)
+}
+
 // makes new boltdb, put two records
 func prep(t *testing.T) *BoltDB {
 	os.Remove(testDb)

app/store/store.go

@@ -60,6 +60,7 @@ type Interface interface {
 	Get(locator Locator, commentID string) (Comment, error)
 	Vote(locator Locator, commentID string, userID string, val bool) (Comment, error)
 	Count(locator Locator) (int, error)
+	List(locator Locator) ([]string, error)
 
 	SetBlock(locator Locator, userID string, status bool) error
 	IsBlocked(locator Locator, userID string) bool