diff --git a/README.md b/README.md
index be6d840a..ad9effd3 100644
--- a/README.md
+++ b/README.md
@@ -63,3 +63,4 @@ type Locator struct {
- `PUT /api/v1/vote/{id}?url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decreases. _auth required_
- `DELETE /api/v1/moderate/comment/{id}?url=post-url` - delete comment by `id`. _auth and admin required_
- `PUT /api/v1/moderate/user/{userid}?site=side-id&block=1` - block or unblock user. _auth and admin required_
+- `GET /api/v1/export?site=side-id&block=1` - export all comments. _auth and admin required_
diff --git a/app/main.go b/app/main.go
index 4c6252c9..5a649326 100644
--- a/app/main.go
+++ b/app/main.go
@@ -72,6 +72,7 @@ func main() {
SessionStore: sessionStore,
Admins: opts.Admins,
DevMode: opts.DevMode,
+ Exporter: &migrator.Remark{DataStore: dataStore},
AuthGoogle: auth.NewGoogle(auth.Params{
Cid: opts.ServerCommand.GoogleCID,
Csecret: opts.ServerCommand.GoogleCSEC,
diff --git a/app/migrator/remark.go b/app/migrator/remark.go
new file mode 100644
index 00000000..5f2a78ff
--- /dev/null
+++ b/app/migrator/remark.go
@@ -0,0 +1,52 @@
+package migrator
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "log"
+
+ "github.com/pkg/errors"
+ "github.com/umputun/remark/app/store"
+)
+
+// Remark implements exporter and importer for internal store
+type Remark struct {
+ DataStore store.Interface
+}
+
+// Export all comments to writer as json
+func (r *Remark) Export(w io.Writer, siteID string) error {
+ topics, err := r.DataStore.List(store.Locator{SiteID: siteID})
+ if err != nil {
+ return err
+ }
+ log.Printf("[DEBUG] exporting %d topics", len(topics))
+
+ commentsCount := 0
+ for _, topic := range topics {
+ comments, err := r.DataStore.Find(store.Request{Locator: store.Locator{SiteID: siteID, URL: topic}})
+ if err != nil {
+ return err
+ }
+
+ for _, comment := range comments {
+
+ buf := &bytes.Buffer{}
+ enc := json.NewEncoder(buf)
+ enc.SetEscapeHTML(false)
+
+ if err := enc.Encode(comment); err != nil {
+ return errors.Wrapf(err, "can't marshal %v", comments)
+ }
+ data := buf.Bytes()
+ data = append(data, '\n')
+ if _, err := w.Write(data); err != nil {
+ return errors.Wrap(err, "can't write comment data")
+ }
+ commentsCount++
+ }
+ }
+ log.Printf("[DEBUG] exported %d comments", commentsCount)
+ return nil
+}
diff --git a/app/migrator/remark_test.go b/app/migrator/remark_test.go
new file mode 100644
index 00000000..fe343f76
--- /dev/null
+++ b/app/migrator/remark_test.go
@@ -0,0 +1,57 @@
+package migrator
+
+import (
+ "bytes"
+ "log"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/umputun/remark/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestRemark_Export(t *testing.T) {
+ b := prep(t)
+ r := Remark{DataStore: b}
+
+ buf := &bytes.Buffer{}
+ err := r.Export(buf, "radio-t")
+ assert.Nil(t, err)
+
+ c1, err := buf.ReadString('\n')
+ assert.Nil(t, err)
+ log.Print(c1)
+ exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
+ assert.Equal(t, exp, c1)
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) *store.BoltDB {
+ os.Remove(testDb)
+
+ b, err := store.NewBoltDB(testDb)
+ assert.Nil(t, err)
+
+ comment := store.Comment{
+ ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ comment = store.Comment{
+ Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ return b
+}
diff --git a/app/rest/server.go b/app/rest/server.go
index db423641..2575ca89 100644
--- a/app/rest/server.go
+++ b/app/rest/server.go
@@ -17,6 +17,7 @@ import (
"bytes"
"encoding/json"
+ "github.com/umputun/remark/app/migrator"
"github.com/umputun/remark/app/rest/auth"
"github.com/umputun/remark/app/store"
)
@@ -29,6 +30,7 @@ type Server struct {
AuthGoogle *auth.Provider
AuthGithub *auth.Provider
SessionStore *sessions.FilesystemStore
+ Exporter migrator.Exporter
DevMode bool
mod moderator
@@ -60,9 +62,11 @@ func (s *Server) Run() {
rauth.Put("/vote/{id}", s.voteCtrl)
})
- rapi.With(Auth(s.SessionStore, s.Admins, s.DevMode)).Group(func(rmoder chi.Router) {
+ rapi.With(Auth(s.SessionStore, s.Admins, s.DevMode)).Group(func(radmin chi.Router) {
s.mod = moderator{dataStore: s.Store}
- rmoder.Mount("/moderate", s.mod.routes())
+ radmin.Get("/export", s.exportCtrl)
+ radmin.Mount("/moderate", s.mod.routes())
+
})
})
@@ -249,6 +253,12 @@ func (s *Server) voteCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, comment)
}
+// GET /export?site=site-id
+func (s *Server) exportCtrl(w http.ResponseWriter, r *http.Request) {
+ siteID := r.URL.Query().Get("site")
+ s.Exporter.Export(w, siteID)
+}
+
func httpError(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
render.Status(r, code)
render.JSON(w, r, JSON{"error": err.Error(), "details": details})
diff --git a/app/store/bolt.go b/app/store/bolt.go
index 99ec881f..e21c8712 100644
--- a/app/store/bolt.go
+++ b/app/store/bolt.go
@@ -325,16 +325,18 @@ func (b *BoltDB) bucketForBlock(locator Locator, userID string) []byte {
return []byte(fmt.Sprintf("%s%s", blocksBucketPrefix, locator.SiteID))
}
-// buckets returns list of buckets, which is list of all commented posts
-func (b BoltDB) buckets() (result []string) {
+// List returns list of buckets, which is list of all commented posts
+func (b BoltDB) List(locator Locator) (result []string, err error) {
- _ = b.View(func(tx *bolt.Tx) error {
+ err = b.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
- result = append(result, string(name))
+ if string(name) != lastBucketName {
+ result = append(result, string(name))
+ }
return nil
})
})
- return result
+ return result, err
}
type ref struct {
diff --git a/app/store/bolt_test.go b/app/store/bolt_test.go
index 0afb20d1..3f36d4e0 100644
--- a/app/store/bolt_test.go
+++ b/app/store/bolt_test.go
@@ -18,7 +18,7 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
res, err := b.Find(Request{Locator: Locator{URL: "https://radio-t.com"}})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text, link", res[0].Text)
+ assert.Equal(t, `some text, link`, res[0].Text)
assert.Equal(t, "user1", res[0].User.ID)
t.Log(res[0].ID)
}
@@ -126,6 +126,21 @@ func TestBoltDB_BlockUser(t *testing.T) {
}
+func TestBoltDB_List(t *testing.T) {
+ defer os.Remove(testDb)
+ b := prep(t) // two comments for https://radio-t.com
+
+ // add one more for https://radio-t.com/2
+ comment := Comment{Text: `some text, link`, Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, User: User{ID: "user1", Name: "user name"}}
+ _, err := b.Create(comment)
+ assert.Nil(t, err)
+
+ res, err := b.List(Locator{SiteID: "site1"})
+ assert.Nil(t, err)
+ assert.Equal(t, []string{"https://radio-t.com", "https://radio-t.com/2"}, res)
+}
+
// makes new boltdb, put two records
func prep(t *testing.T) *BoltDB {
os.Remove(testDb)
diff --git a/app/store/store.go b/app/store/store.go
index f1f2e3ea..2cc0c829 100644
--- a/app/store/store.go
+++ b/app/store/store.go
@@ -60,6 +60,7 @@ type Interface interface {
Get(locator Locator, commentID string) (Comment, error)
Vote(locator Locator, commentID string, userID string, val bool) (Comment, error)
Count(locator Locator) (int, error)
+ List(locator Locator) ([]string, error)
SetBlock(locator Locator, userID string, status bool) error
IsBlocked(locator Locator, userID string) bool