Merge pull request #238 from umputun/migrator-meta

Migrator meta

backend/app/cmd/backup.go

@@ -18,7 +18,7 @@ type BackupCommand struct {
 	ExportPath string        `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
 	ExportFile string        `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
 	Site       string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout    time.Duration `long:"timeout" default:"15m" description:"import timeout"`
+	Timeout    time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"`
 	CommonOpts
 }
 

backend/app/cmd/cleanup.go

@@ -16,7 +16,7 @@ import (
 // CleanupCommand set of flags and command for cleanup
 type CleanupCommand struct {
 	Site     string   `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Dry      bool     `long:"dry" env:"DRY" description:"dry mode, will not remove comments"`
+	Dry      bool     `long:"dry" description:"dry mode, will not remove comments"`
 	From     string   `long:"from" description:"from yyyymmdd"`
 	To       string   `long:"to" description:"from yyyymmdd"`
 	BadWords []string `short:"w" long:"bword" description:"bad word(s)"`

backend/app/cmd/server.go

@@ -230,14 +230,14 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		RemarkURL: strings.TrimSuffix(s.RemarkURL, "/"),
 	}
 
-	exporter := &migrator.Remark{DataStore: dataService}
+	exporter := &migrator.Native{DataStore: dataService}
 
 	migr := &api.Migrator{
 		Cache:             loadingCache,
-		NativeImporter:    &migrator.Remark{DataStore: dataService},
+		NativeImporter:    &migrator.Native{DataStore: dataService},
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
-		NativeExported:    &migrator.Remark{DataStore: dataService},
+		NativeExporter:    &migrator.Native{DataStore: dataService},
 		KeyStore:          adminStore,
 	}
 

backend/app/migrator/migrator.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 // Importer defines interface to convert posts from external sources
@@ -29,6 +30,8 @@ type Store interface {
 	Find(locator store.Locator, sort string) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
+	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
+	SetMetas(siteID string, umetas []service.UserMetaData, pmetas []service.PostMetaData) error
 }
 
 // ImportParams defines everything needed to run import
@@ -50,7 +53,7 @@ func ImportComments(p ImportParams) (int, error) {
 	case "wordpress":
 		importer = &WordPress{DataStore: p.DataStore}
 	case "native":
-		importer = &Remark{DataStore: p.DataStore}
+		importer = &Native{DataStore: p.DataStore}
 	default:
 		return 0, errors.Errorf("unsupported import provider %s", p.Provider)
 	}

backend/app/migrator/migrator_test.go

@@ -66,13 +66,13 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
-func TestMigrator_ImportRemark(t *testing.T) {
+func TestMigrator_ImportNative(t *testing.T) {
 	defer func() {
 		os.Remove("/tmp/remark-test.db")
 		os.Remove("/tmp/disqus-test.r42")
 	}()
 
-	data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
+	data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
 	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)

backend/app/migrator/native.go

@@ -0,0 +1,138 @@
+package migrator
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+const natvieVersion = 1
+
+// Native implements exporter and importer for internal store format
+// {"version": 1, comments:[{...}\n,{}], meta: {meta}}
+// each comments starts from the new line
+type Native struct {
+	DataStore Store
+}
+
+type meta struct {
+	Version int                    `json:"version"`
+	Users   []service.UserMetaData `json:"users"`
+	Posts   []service.PostMetaData `json:"posts"`
+}
+
+// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
+// The final file is a valid json
+func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
+
+	if err = n.exportMeta(siteID, w); err != nil {
+		return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID)
+	}
+
+	topics, err := n.DataStore.List(siteID, 0, 0)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Printf("[DEBUG] exporting %d topics", len(topics))
+	commentsCount := 0
+	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
+		topic := topics[i]
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		if err != nil {
+			return commentsCount, e
+		}
+
+		for _, comment := range comments {
+
+			buf := &bytes.Buffer{}
+			enc := json.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+
+			if err = enc.Encode(comment); err != nil {
+				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
+			}
+			if _, err = w.Write(buf.Bytes()); err != nil {
+				return commentsCount, errors.Wrap(err, "can't write comment data")
+			}
+			commentsCount++
+		}
+	}
+	log.Printf("[DEBUG] exported %d comments", commentsCount)
+	return commentsCount, nil
+}
+
+// exportMeta appends user and post metas to exported stream
+func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
+	m := meta{Version: natvieVersion}
+	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
+	if err != nil {
+		return errors.Wrap(err, "can't get meta")
+	}
+
+	if err := json.NewEncoder(w).Encode(m); err != nil {
+		return errors.Wrap(err, "can't encode meta")
+	}
+	return nil
+}
+
+// Import comments from json strings produced by Remark.Export
+func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
+
+	m := meta{}
+	dec := json.NewDecoder(reader)
+	if err = dec.Decode(&m); err != nil {
+		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
+	}
+
+	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+		return 0, errors.Errorf("unexpected import file version %d", m.Version)
+	}
+
+	if err = n.DataStore.DeleteAll(siteID); err != nil {
+		return 0, err
+	}
+
+	failed := 0
+	total, comments := 0, 0
+
+	for {
+		comment := store.Comment{}
+		err = dec.Decode(&comment)
+		if err == io.EOF {
+			break
+		}
+
+		total++
+
+		if err != nil {
+			failed++
+			continue
+		}
+
+		if _, err = n.DataStore.Create(comment); err != nil {
+			failed++
+			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
+			continue
+		}
+		comments++
+		if comments%1000 == 0 {
+			log.Printf("[DEBUG] imported %d comments", comments)
+		}
+	}
+
+	if failed > 0 {
+		return comments, errors.Errorf("failed to save %d comments", failed)
+	}
+	log.Printf("[INFO] imported %d comments from %d records", comments, total)
+
+	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
+
+	return comments, err
+}

backend/app/migrator/native_test.go

@@ -0,0 +1,166 @@
+package migrator
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestNative_Export(t *testing.T) {
+	defer os.Remove(testDb)
+	b := prep(t) // write 2 comments
+	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
+	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
+	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
+	r := Native{DataStore: b}
+
+	buf := &bytes.Buffer{}
+	size, err := r.Export(buf, "radio-t")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, size)
+
+	c1 := buf.String()
+	log.Print(c1)
+
+	dec := json.NewDecoder(strings.NewReader(c1))
+
+	meta := struct {
+		Version int                    `json:"version"`
+		Users   []service.UserMetaData `json:"users"`
+		Posts   []service.PostMetaData `json:"posts"`
+	}{}
+
+	require.NoError(t, dec.Decode(&meta), "decode meta")
+
+	assert.Equal(t, 2, len(meta.Users))
+	assert.Equal(t, "user1", meta.Users[0].ID)
+	assert.Equal(t, false, meta.Users[0].Blocked.Status)
+	assert.Equal(t, true, meta.Users[0].Verified)
+	assert.Equal(t, "user2", meta.Users[1].ID)
+	assert.Equal(t, true, meta.Users[1].Blocked.Status)
+	assert.Equal(t, false, meta.Users[1].Verified)
+
+	assert.Equal(t, 1, len(meta.Posts))
+	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
+	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+
+	comments := [3]store.Comment{}
+
+	assert.NoError(t, dec.Decode(&comments[0]), "decode comment 0")
+	assert.NoError(t, dec.Decode(&comments[1]), "decode comment 0")
+	assert.Error(t, dec.Decode(&comments[2]), "EOF")
+
+	assert.Equal(t, "some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>", comments[0].Text)
+}
+
+func TestNative_Import(t *testing.T) {
+	defer os.Remove(testDb)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	size, err := r.Import(strings.NewReader(inp), "radio-t")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
+func TestNative_ImportWrongVersion(t *testing.T) {
+	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	size, err := r.Import(strings.NewReader(inp), "radio-t")
+	assert.EqualError(t, err, "unexpected import file version 2")
+	assert.Equal(t, 0, size)
+
+}
+func TestNative_ImportManyWithError(t *testing.T) {
+	defer os.Remove(testDb)
+
+	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
+
+	buf := &bytes.Buffer{}
+	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
+	for i := 0; i < 1200; i++ {
+		buf.WriteString(fmt.Sprintf(goodRec, i))
+	}
+	buf.WriteString("{}\n")
+	buf.WriteString("{}\n")
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	n, err := r.Import(buf, "radio-t")
+	assert.EqualError(t, err, "failed to save 2 comments")
+	assert.Equal(t, 1200, n)
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
+	assert.Nil(t, err)
+	assert.Equal(t, 1200, len(comments))
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) *service.DataStore {
+	os.Remove(testDb)
+
+	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
+	assert.Nil(t, err)
+
+	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+
+	comment := store.Comment{
+		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	comment = store.Comment{
+		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:    store.User{ID: "user2", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	return b
+}

backend/app/migrator/remark.go

@@ -1,96 +0,0 @@
-package migrator
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"io"
-	"log"
-
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Remark implements exporter and importer for internal store format
-type Remark struct {
-	DataStore Store
-}
-
-// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
-func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
-	topics, err := r.DataStore.List(siteID, 0, 0)
-	if err != nil {
-		return 0, err
-	}
-	log.Printf("[DEBUG] exporting %d topics", len(topics))
-
-	commentsCount := 0
-	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
-		topic := topics[i]
-		comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
-			return commentsCount, err
-		}
-
-		for _, comment := range comments {
-
-			buf := &bytes.Buffer{}
-			enc := json.NewEncoder(buf)
-			enc.SetEscapeHTML(false)
-
-			if err := enc.Encode(comment); err != nil {
-				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
-			}
-			data := buf.Bytes()
-			if _, err := w.Write(data); err != nil {
-				return commentsCount, errors.Wrap(err, "can't write comment data")
-			}
-			commentsCount++
-		}
-	}
-	log.Printf("[DEBUG] exported %d comments", commentsCount)
-	return commentsCount, nil
-}
-
-// Import comments from json strings produced by Remark.Export
-func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
-
-	if err := r.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
-	}
-
-	failed := 0
-	total, comments := 0, 0
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		rec := scanner.Bytes()
-		if len(rec) < 3 {
-			continue
-		}
-		total++
-		comment := store.Comment{}
-		if err := json.Unmarshal(rec, &comment); err != nil {
-			failed++
-			log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
-			continue
-		}
-		if _, err := r.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
-	}
-	if scanner.Err() != nil {
-		return comments, errors.Wrap(scanner.Err(), "error in scan")
-	}
-	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
-	}
-	log.Printf("[INFO] imported %d comments from %d records", comments, total)
-	return comments, nil
-}

backend/app/migrator/remark_test.go

@@ -1,123 +0,0 @@
-package migrator
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestRemark_Export(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	r := Remark{DataStore: b}
-
-	buf := &bytes.Buffer{}
-	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	c1, err := buf.ReadString('\n')
-	assert.Nil(t, err)
-	log.Print(c1)
-	exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-	assert.Equal(t, exp, c1)
-}
-
-func TestRemark_Import(t *testing.T) {
-	defer os.Remove(testDb)
-
-	r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	buf.WriteString(r1)
-	buf.WriteString(r2)
-	buf.WriteString("{}")
-
-	b := prep(t) // write some recs
-	r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
-	size, err := r.Import(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[0].ID)
-	assert.Equal(t, "afbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ParentID)
-
-	// try import again
-	buf.WriteString(r1)
-	buf.WriteString(r2)
-	buf.WriteString("{}")
-	size, err = r.Import(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-}
-
-func TestRemark_ImportManyWithError(t *testing.T) {
-	defer os.Remove(testDb)
-
-	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < 1200; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
-	}
-	buf.WriteString("bad1\n")
-	buf.WriteString("bad2\n")
-
-	b := prep(t) // write some recs
-	r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
-	n, err := r.Import(buf, "radio-t")
-	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
-
-	comment := store.Comment{
-		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

backend/app/rest/api/admin_test.go

@@ -455,7 +455,7 @@ func TestAdmin_ExportStream(t *testing.T) {
 
 	body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
 	assert.Equal(t, 200, code)
-	assert.Equal(t, 2, strings.Count(body, "\n"))
+	assert.Equal(t, 3, strings.Count(body, "\n"))
 	assert.Equal(t, 2, strings.Count(body, "\"text\""))
 	t.Logf("%s", body)
 }
@@ -487,7 +487,7 @@ func TestAdmin_ExportFile(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
+	assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
 	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
 	t.Logf("%s", string(ungzBody))
 }

backend/app/rest/api/migrator.go

@@ -27,7 +27,7 @@ type Migrator struct {
 	NativeImporter    migrator.Importer
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
-	NativeExported    migrator.Exporter
+	NativeExporter    migrator.Exporter
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -153,7 +153,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		writer = gzWriter
 	}
 
-	if _, err := m.NativeExported.Export(writer, siteID); err != nil {
+	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
 		return
 	}

backend/app/rest/api/migrator_test.go

@@ -34,7 +34,7 @@ func TestMigrator_Import(t *testing.T) {
 	assert.NotNil(t, srv)
 	defer cleanupImportSrv(srv, ts)
 
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
@@ -61,7 +61,7 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.NotNil(t, srv)
 	defer cleanupImportSrv(srv, ts)
 
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	bodyBuf := &bytes.Buffer{}
@@ -136,7 +136,7 @@ func TestMigrator_ImportRejected(t *testing.T) {
 	assert.NotNil(t, srv)
 	defer cleanupImportSrv(srv, ts)
 
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
@@ -157,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	for i := 0; i < 1000; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
-	r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r)
 	assert.Nil(t, err)
@@ -184,7 +184,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	for i := 0; i < 1000; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
-	r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r)
 	require.Nil(t, err)
@@ -205,7 +205,7 @@ func TestMigrator_Export(t *testing.T) {
 	assert.NotNil(t, srv)
 	defer cleanupImportSrv(srv, ts)
 
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
@@ -234,7 +234,7 @@ func TestMigrator_Export(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
+	assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
 	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
 	t.Logf("%s", string(ungzBody))
 
@@ -248,7 +248,7 @@ func TestMigrator_Export(t *testing.T) {
 
 	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(body), "\n"))
+	assert.Equal(t, 3, strings.Count(string(body), "\n"))
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
@@ -267,8 +267,8 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http
 	svc = &Migrator{
 		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
 		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Remark{DataStore: dataStore},
-		NativeExported:    &migrator.Remark{DataStore: dataStore},
+		NativeImporter:    &migrator.Native{DataStore: dataStore},
+		NativeExporter:    &migrator.Native{DataStore: dataStore},
 		Cache:             &cache.Nop{},
 		KeyStore:          adminStore,
 	}

backend/app/rest/api/rest.go

@@ -142,12 +142,12 @@ func (s *Rest) Shutdown() {
 	s.lock.Unlock()
 }
 
-func (s *Rest) makeHTTPServer(port int, router chi.Router) *http.Server {
+func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 	return &http.Server{
 		Addr:              fmt.Sprintf(":%d", port),
 		Handler:           router,
 		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      5 * time.Second,
+		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
 		IdleTimeout:       30 * time.Second,
 	}
 }

backend/app/rest/api/rest_test.go

@@ -209,8 +209,8 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
 		Migrator: &Migrator{
 			DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-			NativeImporter:    &migrator.Remark{DataStore: dataStore},
-			NativeExported:    &migrator.Remark{DataStore: dataStore},
+			NativeImporter:    &migrator.Native{DataStore: dataStore},
+			NativeExporter:    &migrator.Native{DataStore: dataStore},
 			Cache:             &cache.Nop{},
 			KeyStore:          adminStore,
 		},

backend/app/rest/api/ssl.go

@@ -82,7 +82,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager {
 }
 
 // makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
-func (s *Rest) makeHTTPSAutocertServer(port int, router chi.Router, m *autocert.Manager) *http.Server {
+func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
 	server := s.makeHTTPServer(port, router)
 	cfg := makeTLSConfig()
 	cfg.GetCertificate = m.GetCertificate
@@ -91,7 +91,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router chi.Router, m *autocert.
 }
 
 // makeHTTPSServer makes https server for static mode
-func (s *Rest) makeHTTPSServer(port int, router chi.Router) *http.Server {
+func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
 	server := s.makeHTTPServer(port, router)
 	server.TLSConfig = makeTLSConfig()
 	return server

backend/app/store/engine/bolt_accessor.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/coreos/bbolt"
-	"github.com/hashicorp/go-multierror"
+	bolt "github.com/coreos/bbolt"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"

backend/app/store/engine/bolt_admin.go

@@ -320,3 +320,20 @@ func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
 	})
 	return verified
 }
+
+// Verified returns list of verified userIDs
+func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return nil, err
+	}
+	err = bdb.View(func(tx *bolt.Tx) error {
+		usersBkt := tx.Bucket([]byte(verifiedBucketName))
+		_ = usersBkt.ForEach(func(k, _ []byte) error {
+			ids = append(ids, string(k))
+			return nil
+		})
+		return nil
+	})
+	return ids, err
+}

backend/app/store/engine/bolt_admin_test.go

@@ -214,4 +214,15 @@ func TestBoltAdmin_Verified(t *testing.T) {
 	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
 
 	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
+	assert.NoError(t, b.SetVerified("radio-t", "u2", true))
+	assert.NoError(t, b.SetVerified("radio-t", "u3", false))
+
+	ids, err := b.Verified("radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
+
+	_, err = b.Verified("radio-t-bad")
+	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
 }

backend/app/store/engine/engine.go

@@ -53,6 +53,7 @@ type Admin interface {
 	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
 	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
 	IsVerified(siteID string, userID string) bool                                // check verified status
+	Verified(siteID string) ([]string, error)                                    // list of verified user ids
 }
 
 const (

backend/app/store/engine/mongo.go

@@ -6,7 +6,7 @@ import (
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"github.com/go-pkgz/mongo"
-	"github.com/hashicorp/go-multierror"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -221,6 +221,21 @@ func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
 	return err == nil && meta.Verified
 }
 
+// Verified returns list of verified user IDs
+func (m *Mongo) Verified(siteID string) (ids []string, err error) {
+	metas := []metaUser{}
+	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+		return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
+	})
+	if err != nil {
+		return nil, err
+	}
+	for _, m := range metas {
+		ids = append(ids, m.ID)
+	}
+	return ids, nil
+}
+
 // SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
 // block uses blocksBucketName with key=userID and val=TTL+now
 func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {

backend/app/store/engine/mongo_test.go

@@ -239,6 +239,18 @@ func TestMongo_Verified(t *testing.T) {
 	assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
 
 	assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, m.SetVerified("radio-t", "u1", true))
+	assert.NoError(t, m.SetVerified("radio-t", "u2", true))
+	assert.NoError(t, m.SetVerified("radio-t", "u3", false))
+
+	ids, err := m.Verified("radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
+
+	ids, err = m.Verified("radio-t-bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
 }
 
 func TestMongo_GetForUser(t *testing.T) {

backend/app/store/service/service.go

@@ -1,9 +1,12 @@
 package service
 
 import (
+	"sort"
 	"sync"
 	"time"
 
+	multierror "github.com/hashicorp/go-multierror"
+
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
@@ -28,6 +31,22 @@ type DataStore struct {
 	}
 }
 
+// UserMetaData keeps info about user flags
+type UserMetaData struct {
+	ID      string `json:"id"`
+	Blocked struct {
+		Status bool      `json:"status"`
+		Until  time.Time `json:"until"`
+	} `json:"blocked"`
+	Verified bool `json:"verified"`
+}
+
+// PostMetaData keeps info about post flags
+type PostMetaData struct {
+	URL      string `json:"url"`
+	ReadOnly bool   `json:"read_only"`
+}
+
 const defaultCommentMaxSize = 2000
 
 // UnlimitedVotes doesn't restrict MaxVotes
@@ -206,6 +225,86 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
 	return false
 }
 
+// Metas returns metadata for users and posts
+func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
+	umetas = []UserMetaData{}
+	pmetas = []PostMetaData{}
+	// set posts meta
+	posts, err := s.List(siteID, 0, 0)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID)
+	}
+	for _, p := range posts {
+		if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) {
+			pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true})
+		}
+
+	}
+
+	// set users meta
+	m := map[string]UserMetaData{}
+
+	// process blocked users
+	blocked, err := s.Blocked(siteID)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID)
+	}
+	for _, b := range blocked {
+		val, ok := m[b.ID]
+		if !ok {
+			val = UserMetaData{ID: b.ID}
+		}
+		val.Blocked.Status = true
+		val.Blocked.Until = b.Until
+		m[b.ID] = val
+	}
+
+	// process verified users
+	verified, err := s.Verified(siteID)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID)
+	}
+	for _, v := range verified {
+		val, ok := m[v]
+		if !ok {
+			val = UserMetaData{ID: v}
+		}
+		val.Verified = true
+		m[v] = val
+	}
+
+	for _, u := range m {
+		umetas = append(umetas, u)
+	}
+	sort.Slice(umetas, func(i, j int) bool { return umetas[i].ID < umetas[j].ID })
+
+	return umetas, pmetas, nil
+}
+
+// SetMetas saves metadata for users and posts
+func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []PostMetaData) (err error) {
+	errs := new(multierror.Error)
+
+	// save posts metas
+	for _, pm := range pmetas {
+		if pm.ReadOnly {
+			errs = multierror.Append(errs, s.SetReadOnly(store.Locator{SiteID: siteID, URL: pm.URL}, true))
+		}
+	}
+
+	// save users metas
+	for _, um := range umetas {
+		if um.Blocked.Status {
+			errs = multierror.Append(errs, s.SetBlock(siteID, um.ID, true, time.Until(um.Blocked.Until)))
+		}
+		if um.Verified {
+			errs = multierror.Append(errs, s.SetVerified(siteID, um.ID, true))
+		}
+	}
+
+	return errs.ErrorOrNil()
+}
+
 // getsScopedLocks pull lock from the map if found or create a new one
 func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })

backend/app/store/service/service_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -374,6 +374,57 @@ func TestService_Counts(t *testing.T) {
 	}, res)
 }
 
+func TestService_GetMetas(t *testing.T) {
+	defer os.Remove(testDb)
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123")}
+
+	um, pm, err := b.Metas("radio-t")
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(um))
+	assert.Equal(t, 0, len(pm))
+
+	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
+	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
+
+	um, pm, err = b.Metas("radio-t")
+	require.NoError(t, err)
+
+	assert.Equal(t, 1, len(um))
+	assert.Equal(t, "user1", um[0].ID)
+	assert.Equal(t, true, um[0].Verified)
+	assert.Equal(t, false, um[0].Blocked.Status)
+
+	assert.Equal(t, 1, len(pm))
+	assert.Equal(t, "https://radio-t.com", pm[0].URL)
+	assert.Equal(t, true, pm[0].ReadOnly)
+}
+
+func TestService_SetMetas(t *testing.T) {
+	defer os.Remove(testDb)
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123")}
+	umetas := []UserMetaData{}
+	pmetas := []PostMetaData{}
+	err := b.SetMetas("radio-t", umetas, pmetas)
+	assert.NoError(t, err, "empty metas")
+
+	um1 := UserMetaData{ID: "user1", Verified: true}
+	um2 := UserMetaData{ID: "user2"}
+	um2.Blocked.Status = true
+	um2.Blocked.Until = time.Now().AddDate(0, 1, 1)
+
+	pmetas = []PostMetaData{{URL: "https://radio-t.com", ReadOnly: true}}
+	err = b.SetMetas("radio-t", []UserMetaData{um1, um2}, pmetas)
+	assert.NoError(t, err)
+
+	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}))
+	assert.True(t, b.IsVerified("radio-t", "user1"))
+	assert.True(t, b.IsBlocked("radio-t", "user2"))
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
 	os.Remove(testDb)