mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change native export to new format

Browse Source
This commit is contained in:
Umputun
2018-12-21 00:01:59 -06:00
parent 10f21f78f0
commit 110566a658
5 changed files with 100 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/migrator/migrator.go
+2
View File
@@ -11,6 +11,7 @@ import (
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
// Importer defines interface to convert posts from external sources
@@ -29,6 +30,7 @@ type Store interface {
Find(locator store.Locator, sort string) ([]store.Comment, error)
List(siteID string, limit int, skip int) ([]store.PostInfo, error)
DeleteAll(siteID string) error
Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
}
// ImportParams defines everything needed to run import
backend/app/migrator/remark.go
+45 -2
View File
@@ -4,27 +4,41 @@ import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
const (
header = `{"version":1, "comments":[`
metaHeader = "],\n\"meta\":"
footer = `}`
)
// Remark implements exporter and importer for internal store format
// {"version": 1, comments:[{...}\n,{}], meta: {meta}}
type Remark struct {
DataStore Store
}
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
if _, err := fmt.Fprintf(w, "%s\n", header); err != nil {
return 0, err
}
topics, err := r.DataStore.List(siteID, 0, 0)
if err != nil {
return 0, err
}
log.Printf("[DEBUG] exporting %d topics", len(topics))
log.Printf("[DEBUG] exporting %d topics", len(topics))
commentsCount := 0
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
topic := topics[i]
@@ -33,7 +47,7 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
return commentsCount, err
}
for _, comment := range comments {
for n, comment := range comments {
buf := &bytes.Buffer{}
enc := json.NewEncoder(buf)
@@ -43,13 +57,42 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
}
data := buf.Bytes()
data = bytes.TrimSuffix(data, []byte("\n"))
if _, err := w.Write(data); err != nil {
return commentsCount, errors.Wrap(err, "can't write comment data")
}
if n < len(comments)-1 || i != 0 { // don't add , on last comment
w.Write([]byte(","))
}
w.Write([]byte("\n"))
commentsCount++
}
}
log.Printf("[DEBUG] exported %d comments", commentsCount)
if _, err := fmt.Fprintf(w, "%s", metaHeader); err != nil {
return 0, err
}
meta := struct {
Users []service.UserMetaData `json:"users"`
Posts []service.PostMetaData `json:"posts"`
}{}
meta.Users, meta.Posts, err = r.DataStore.Metas(siteID)
if err != nil {
return 0, err
}
if err := json.NewEncoder(w).Encode(meta); err != nil {
return 0, err
}
if _, err := fmt.Fprintf(w, "%s\n", footer); err != nil {
return 0, err
}
return commentsCount, nil
}
backend/app/migrator/remark_test.go
+35 -7
View File
@@ -2,13 +2,14 @@ package migrator
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
"testing"
"time"
"github.com/coreos/bbolt"
bolt "github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store"
@@ -22,6 +23,9 @@ var testDb = "/tmp/test-remark.db"
func TestRemark_Export(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true)
b.SetBlock("radio-t", "user-2", true, time.Hour)
b.SetVerified("radio-t", "user-1", true)
r := Remark{DataStore: b}
buf := &bytes.Buffer{}
@@ -29,11 +33,35 @@ func TestRemark_Export(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, size)
c1, err := buf.ReadString('\n')
assert.Nil(t, err)
c1 := buf.String()
log.Print(c1)
exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
assert.Equal(t, exp, c1)
res := struct {
Version int `json:"version"`
Comments []store.Comment `json:"comments"`
Meta struct {
Users []service.UserMetaData `json:"users"`
Posts []service.PostMetaData `json:"posts"`
} `json:"meta"`
}{}
err = json.Unmarshal([]byte(c1), &res)
assert.NoError(t, err)
assert.Equal(t, 2, len(res.Comments))
assert.Equal(t, "some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>", res.Comments[0].Text)
assert.Equal(t, 2, len(res.Meta.Users))
assert.Equal(t, "user-2", res.Meta.Users[0].ID)
assert.Equal(t, true, res.Meta.Users[0].Blocked.Status)
assert.Equal(t, false, res.Meta.Users[0].Verified)
assert.Equal(t, "user-1", res.Meta.Users[1].ID)
assert.Equal(t, false, res.Meta.Users[1].Blocked.Status)
assert.Equal(t, true, res.Meta.Users[1].Verified)
assert.Equal(t, 1, len(res.Meta.Posts))
assert.Equal(t, "https://radio-t.com", res.Meta.Posts[0].URL)
assert.Equal(t, true, res.Meta.Posts[0].ReadOnly)
}
func TestRemark_Import(t *testing.T) {
@@ -113,8 +141,8 @@ func prep(t *testing.T) *service.DataStore {
comment = store.Comment{
Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user2", Name: "user name"},
}
_, err = b.Create(comment)
assert.Nil(t, err)
backend/app/store/service/service.go
+11 -10
View File
@@ -30,18 +30,18 @@ type DataStore struct {
// UserMetaData keeps info about user flags
type UserMetaData struct {
UserID string
ID string `json:"id"`
Blocked struct {
Status bool
Until time.Time
}
Verified bool
Status bool `json:"status"`
Until time.Time `json:"until"`
} `json:"blocked"`
Verified bool `json:"verified"`
}
// PostMetaData keeps info about post flags
type PostMetaData struct {
URL string
ReadOnly bool
URL string `json:"url"`
ReadOnly bool `json:"read_only"`
}
const defaultCommentMaxSize = 2000
@@ -224,7 +224,8 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
// Metas returns metadata for users and posts
func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
umetas = []UserMetaData{}
pmetas = []PostMetaData{}
// set posts meta
posts, err := s.List(siteID, 0, 0)
if err != nil {
@@ -248,7 +249,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
for _, b := range blocked {
val, ok := m[b.ID]
if !ok {
val = UserMetaData{UserID: b.ID}
val = UserMetaData{ID: b.ID}
}
val.Blocked.Status = true
val.Blocked.Until = b.Until
@@ -263,7 +264,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
for _, v := range verified {
val, ok := m[v]
if !ok {
val = UserMetaData{UserID: v}
val = UserMetaData{ID: v}
}
val.Verified = true
m[v] = val
backend/app/store/service/service_test.go
+7 -2
View File
@@ -9,7 +9,7 @@ import (
"testing"
"time"
"github.com/coreos/bbolt"
bolt "github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -380,10 +380,15 @@ func TestService_Metas(t *testing.T) {
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticKeyStore("secret 123")}
um, pm, err := b.Metas("radio-t")
require.NoError(t, err)
assert.Equal(t, 0, len(um))
assert.Equal(t, 0, len(pm))
assert.NoError(t, b.SetVerified("radio-t", "user1", true))
assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
um, pm, err := b.Metas("radio-t")
um, pm, err = b.Metas("radio-t")
require.NoError(t, err)
assert.Equal(t, 1, len(um))