diff --git a/backend/app/cmd/backup.go b/backend/app/cmd/backup.go index 3a41944e..584dab33 100644 --- a/backend/app/cmd/backup.go +++ b/backend/app/cmd/backup.go @@ -18,7 +18,7 @@ type BackupCommand struct { ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"` ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"` Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` - Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"` + Timeout time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"` CommonOpts } diff --git a/backend/app/cmd/cleanup.go b/backend/app/cmd/cleanup.go index 3b1119dc..f1211bad 100644 --- a/backend/app/cmd/cleanup.go +++ b/backend/app/cmd/cleanup.go @@ -16,7 +16,7 @@ import ( // CleanupCommand set of flags and command for cleanup type CleanupCommand struct { Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"` - Dry bool `long:"dry" env:"DRY" description:"dry mode, will not remove comments"` + Dry bool `long:"dry" description:"dry mode, will not remove comments"` From string `long:"from" description:"from yyyymmdd"` To string `long:"to" description:"from yyyymmdd"` BadWords []string `short:"w" long:"bword" description:"bad word(s)"` diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index ab7abae0..16bc019a 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -230,14 +230,14 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { RemarkURL: strings.TrimSuffix(s.RemarkURL, "/"), } - exporter := &migrator.Remark{DataStore: dataService} + exporter := &migrator.Native{DataStore: dataService} migr := &api.Migrator{ Cache: loadingCache, - NativeImporter: &migrator.Remark{DataStore: dataService}, + NativeImporter: &migrator.Native{DataStore: dataService}, DisqusImporter: &migrator.Disqus{DataStore: dataService}, WordPressImporter: &migrator.WordPress{DataStore: dataService}, - NativeExported: &migrator.Remark{DataStore: dataService}, + NativeExporter: &migrator.Native{DataStore: dataService}, KeyStore: adminStore, } diff --git a/backend/app/migrator/migrator.go b/backend/app/migrator/migrator.go index ae80a52b..2737c278 100644 --- a/backend/app/migrator/migrator.go +++ b/backend/app/migrator/migrator.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/service" ) // Importer defines interface to convert posts from external sources @@ -29,6 +30,8 @@ type Store interface { Find(locator store.Locator, sort string) ([]store.Comment, error) List(siteID string, limit int, skip int) ([]store.PostInfo, error) DeleteAll(siteID string) error + Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error) + SetMetas(siteID string, umetas []service.UserMetaData, pmetas []service.PostMetaData) error } // ImportParams defines everything needed to run import @@ -50,7 +53,7 @@ func ImportComments(p ImportParams) (int, error) { case "wordpress": importer = &WordPress{DataStore: p.DataStore} case "native": - importer = &Remark{DataStore: p.DataStore} + importer = &Native{DataStore: p.DataStore} default: return 0, errors.Errorf("unsupported import provider %s", p.Provider) } diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 42367faf..ff3bf1ca 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -66,13 +66,13 @@ func TestMigrator_ImportWordPress(t *testing.T) { assert.Equal(t, 3, len(last), "3 comments imported") } -func TestMigrator_ImportRemark(t *testing.T) { +func TestMigrator_ImportNative(t *testing.T) { defer func() { os.Remove("/tmp/remark-test.db") os.Remove("/tmp/disqus-test.r42") }() - data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + + data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n" err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600) diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go new file mode 100644 index 00000000..79a5b342 --- /dev/null +++ b/backend/app/migrator/native.go @@ -0,0 +1,138 @@ +package migrator + +import ( + "bytes" + "encoding/json" + "io" + "log" + + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/service" +) + +const natvieVersion = 1 + +// Native implements exporter and importer for internal store format +// {"version": 1, comments:[{...}\n,{}], meta: {meta}} +// each comments starts from the new line +type Native struct { + DataStore Store +} + +type meta struct { + Version int `json:"version"` + Users []service.UserMetaData `json:"users"` + Posts []service.PostMetaData `json:"posts"` +} + +// Export all comments to writer as json strings. Each comment is one string, separated by "\n" +// The final file is a valid json +func (n *Native) Export(w io.Writer, siteID string) (size int, err error) { + + if err = n.exportMeta(siteID, w); err != nil { + return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID) + } + + topics, err := n.DataStore.List(siteID, 0, 0) + if err != nil { + return 0, err + } + + log.Printf("[DEBUG] exporting %d topics", len(topics)) + commentsCount := 0 + for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction + topic := topics[i] + comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") + if err != nil { + return commentsCount, e + } + + for _, comment := range comments { + + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + + if err = enc.Encode(comment); err != nil { + return commentsCount, errors.Wrapf(err, "can't marshal %v", comments) + } + if _, err = w.Write(buf.Bytes()); err != nil { + return commentsCount, errors.Wrap(err, "can't write comment data") + } + commentsCount++ + } + } + log.Printf("[DEBUG] exported %d comments", commentsCount) + return commentsCount, nil +} + +// exportMeta appends user and post metas to exported stream +func (n *Native) exportMeta(siteID string, w io.Writer) (err error) { + m := meta{Version: natvieVersion} + m.Users, m.Posts, err = n.DataStore.Metas(siteID) + if err != nil { + return errors.Wrap(err, "can't get meta") + } + + if err := json.NewEncoder(w).Encode(m); err != nil { + return errors.Wrap(err, "can't encode meta") + } + return nil +} + +// Import comments from json strings produced by Remark.Export +func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { + + m := meta{} + dec := json.NewDecoder(reader) + if err = dec.Decode(&m); err != nil { + return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID) + } + + if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version + return 0, errors.Errorf("unexpected import file version %d", m.Version) + } + + if err = n.DataStore.DeleteAll(siteID); err != nil { + return 0, err + } + + failed := 0 + total, comments := 0, 0 + + for { + comment := store.Comment{} + err = dec.Decode(&comment) + if err == io.EOF { + break + } + + total++ + + if err != nil { + failed++ + continue + } + + if _, err = n.DataStore.Create(comment); err != nil { + failed++ + log.Printf("[WARN] can't write %+v to store, %s", comment, err) + continue + } + comments++ + if comments%1000 == 0 { + log.Printf("[DEBUG] imported %d comments", comments) + } + } + + if failed > 0 { + return comments, errors.Errorf("failed to save %d comments", failed) + } + log.Printf("[INFO] imported %d comments from %d records", comments, total) + + err = n.DataStore.SetMetas(siteID, m.Users, m.Posts) + + return comments, err +} diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go new file mode 100644 index 00000000..9aeccd56 --- /dev/null +++ b/backend/app/migrator/native_test.go @@ -0,0 +1,166 @@ +package migrator + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "strings" + "testing" + "time" + + bolt "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/admin" + "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/service" +) + +var testDb = "/tmp/test-remark.db" + +func TestNative_Export(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) // write 2 comments + assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true)) + assert.NoError(t, b.SetVerified("radio-t", "user1", true)) + assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour)) + r := Native{DataStore: b} + + buf := &bytes.Buffer{} + size, err := r.Export(buf, "radio-t") + assert.Nil(t, err) + assert.Equal(t, 2, size) + + c1 := buf.String() + log.Print(c1) + + dec := json.NewDecoder(strings.NewReader(c1)) + + meta := struct { + Version int `json:"version"` + Users []service.UserMetaData `json:"users"` + Posts []service.PostMetaData `json:"posts"` + }{} + + require.NoError(t, dec.Decode(&meta), "decode meta") + + assert.Equal(t, 2, len(meta.Users)) + assert.Equal(t, "user1", meta.Users[0].ID) + assert.Equal(t, false, meta.Users[0].Blocked.Status) + assert.Equal(t, true, meta.Users[0].Verified) + assert.Equal(t, "user2", meta.Users[1].ID) + assert.Equal(t, true, meta.Users[1].Blocked.Status) + assert.Equal(t, false, meta.Users[1].Verified) + + assert.Equal(t, 1, len(meta.Posts)) + assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL) + assert.Equal(t, true, meta.Posts[0].ReadOnly) + + comments := [3]store.Comment{} + + assert.NoError(t, dec.Decode(&comments[0]), "decode comment 0") + assert.NoError(t, dec.Decode(&comments[1]), "decode comment 0") + assert.Error(t, dec.Decode(&comments[2]), "EOF") + + assert.Equal(t, "some text, link", comments[0].Text) +} + +func TestNative_Import(t *testing.T) { + defer os.Remove(testDb) + + inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} + {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} + {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + + b := prep(t) // write some recs + r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} + size, err := r.Import(strings.NewReader(inp), "radio-t") + assert.Nil(t, err) + assert.Equal(t, 2, size) + + comments, err := b.Last("radio-t", 10) + assert.Nil(t, err) + assert.Equal(t, 2, len(comments)) + assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID) + assert.Equal(t, "1234", comments[0].ParentID) + assert.Equal(t, false, b.IsReadOnly(comments[0].Locator)) + + assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID) + assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL) + assert.Equal(t, true, b.IsReadOnly(comments[1].Locator)) + + assert.Equal(t, false, b.IsBlocked("radio-t", "user1")) + assert.Equal(t, true, b.IsVerified("radio-t", "user1")) + + assert.Equal(t, true, b.IsBlocked("radio-t", "user2")) + assert.Equal(t, false, b.IsVerified("radio-t", "user2")) +} + +func TestNative_ImportWrongVersion(t *testing.T) { + inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} + {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} + {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + + b := prep(t) // write some recs + r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} + size, err := r.Import(strings.NewReader(inp), "radio-t") + assert.EqualError(t, err, "unexpected import file version 2") + assert.Equal(t, 0, size) + +} +func TestNative_ImportManyWithError(t *testing.T) { + defer os.Remove(testDb) + + goodRec := `{"id":"%d","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" + + buf := &bytes.Buffer{} + buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n") + for i := 0; i < 1200; i++ { + buf.WriteString(fmt.Sprintf(goodRec, i)) + } + buf.WriteString("{}\n") + buf.WriteString("{}\n") + + b := prep(t) // write some recs + r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} + n, err := r.Import(buf, "radio-t") + assert.EqualError(t, err, "failed to save 2 comments") + assert.Equal(t, 1200, n) + comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time") + assert.Nil(t, err) + assert.Equal(t, 1200, len(comments)) +} + +// makes new boltdb, put two records +func prep(t *testing.T) *service.DataStore { + os.Remove(testDb) + + boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) + assert.Nil(t, err) + + b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + + comment := store.Comment{ + ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + comment = store.Comment{ + Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + return b +} diff --git a/backend/app/migrator/remark.go b/backend/app/migrator/remark.go deleted file mode 100644 index 93b1505d..00000000 --- a/backend/app/migrator/remark.go +++ /dev/null @@ -1,96 +0,0 @@ -package migrator - -import ( - "bufio" - "bytes" - "encoding/json" - "io" - "log" - - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// Remark implements exporter and importer for internal store format -type Remark struct { - DataStore Store -} - -// Export all comments to writer as json strings. Each comment is one string, separated by "\n" -func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) { - topics, err := r.DataStore.List(siteID, 0, 0) - if err != nil { - return 0, err - } - log.Printf("[DEBUG] exporting %d topics", len(topics)) - - commentsCount := 0 - for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction - topic := topics[i] - comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") - if err != nil { - return commentsCount, err - } - - for _, comment := range comments { - - buf := &bytes.Buffer{} - enc := json.NewEncoder(buf) - enc.SetEscapeHTML(false) - - if err := enc.Encode(comment); err != nil { - return commentsCount, errors.Wrapf(err, "can't marshal %v", comments) - } - data := buf.Bytes() - if _, err := w.Write(data); err != nil { - return commentsCount, errors.Wrap(err, "can't write comment data") - } - commentsCount++ - } - } - log.Printf("[DEBUG] exported %d comments", commentsCount) - return commentsCount, nil -} - -// Import comments from json strings produced by Remark.Export -func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) { - - if err := r.DataStore.DeleteAll(siteID); err != nil { - return 0, err - } - - failed := 0 - total, comments := 0, 0 - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - rec := scanner.Bytes() - if len(rec) < 3 { - continue - } - total++ - comment := store.Comment{} - if err := json.Unmarshal(rec, &comment); err != nil { - failed++ - log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err) - continue - } - if _, err := r.DataStore.Create(comment); err != nil { - failed++ - log.Printf("[WARN] can't write %+v to store, %s", comment, err) - continue - } - comments++ - if comments%1000 == 0 { - log.Printf("[DEBUG] imported %d comments", comments) - } - } - if scanner.Err() != nil { - return comments, errors.Wrap(scanner.Err(), "error in scan") - } - if failed > 0 { - return comments, errors.Errorf("failed to save %d comments", failed) - } - log.Printf("[INFO] imported %d comments from %d records", comments, total) - return comments, nil -} diff --git a/backend/app/migrator/remark_test.go b/backend/app/migrator/remark_test.go deleted file mode 100644 index 7819466a..00000000 --- a/backend/app/migrator/remark_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package migrator - -import ( - "bytes" - "fmt" - "log" - "os" - "testing" - "time" - - "github.com/coreos/bbolt" - "github.com/stretchr/testify/assert" - - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/service" -) - -var testDb = "/tmp/test-remark.db" - -func TestRemark_Export(t *testing.T) { - defer os.Remove(testDb) - b := prep(t) - r := Remark{DataStore: b} - - buf := &bytes.Buffer{} - size, err := r.Export(buf, "radio-t") - assert.Nil(t, err) - assert.Equal(t, 2, size) - - c1, err := buf.ReadString('\n') - assert.Nil(t, err) - log.Print(c1) - exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" - assert.Equal(t, exp, c1) -} - -func TestRemark_Import(t *testing.T) { - defer os.Remove(testDb) - - r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" - - r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n" - - buf := &bytes.Buffer{} - buf.WriteString(r1) - buf.WriteString(r2) - buf.WriteString("{}") - - b := prep(t) // write some recs - r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} - size, err := r.Import(buf, "radio-t") - assert.Nil(t, err) - assert.Equal(t, 2, size) - - comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(comments)) - assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[0].ID) - assert.Equal(t, "afbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID) - assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ParentID) - - // try import again - buf.WriteString(r1) - buf.WriteString(r2) - buf.WriteString("{}") - size, err = r.Import(buf, "radio-t") - assert.Nil(t, err) - assert.Equal(t, 2, size) -} - -func TestRemark_ImportManyWithError(t *testing.T) { - defer os.Remove(testDb) - - goodRec := `{"id":"%d","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" - - buf := &bytes.Buffer{} - for i := 0; i < 1200; i++ { - buf.WriteString(fmt.Sprintf(goodRec, i)) - } - buf.WriteString("bad1\n") - buf.WriteString("bad2\n") - - b := prep(t) // write some recs - r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}} - n, err := r.Import(buf, "radio-t") - assert.EqualError(t, err, "failed to save 2 comments") - assert.Equal(t, 1200, n) - comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time") - assert.Nil(t, err) - assert.Equal(t, 1200, len(comments)) -} - -// makes new boltdb, put two records -func prep(t *testing.T) *service.DataStore { - os.Remove(testDb) - - boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) - assert.Nil(t, err) - - b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} - - comment := store.Comment{ - ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - comment = store.Comment{ - Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - return b -} diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 5a45573d..8258cadc 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -455,7 +455,7 @@ func TestAdmin_ExportStream(t *testing.T) { body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream") assert.Equal(t, 200, code) - assert.Equal(t, 2, strings.Count(body, "\n")) + assert.Equal(t, 3, strings.Count(body, "\n")) assert.Equal(t, 2, strings.Count(body, "\"text\"")) t.Logf("%s", body) } @@ -487,7 +487,7 @@ func TestAdmin_ExportFile(t *testing.T) { assert.NoError(t, err) ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) - assert.Equal(t, 2, strings.Count(string(ungzBody), "\n")) + assert.Equal(t, 3, strings.Count(string(ungzBody), "\n")) assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) } diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go index 2fdf0e4e..080e5b72 100644 --- a/backend/app/rest/api/migrator.go +++ b/backend/app/rest/api/migrator.go @@ -27,7 +27,7 @@ type Migrator struct { NativeImporter migrator.Importer DisqusImporter migrator.Importer WordPressImporter migrator.Importer - NativeExported migrator.Exporter + NativeExporter migrator.Exporter KeyStore KeyStore busy map[string]bool @@ -153,7 +153,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) { writer = gzWriter } - if _, err := m.NativeExported.Export(writer, siteID); err != nil { + if _, err := m.NativeExporter.Export(writer, siteID); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } diff --git a/backend/app/rest/api/migrator_test.go b/backend/app/rest/api/migrator_test.go index dec91a4d..dced318c 100644 --- a/backend/app/rest/api/migrator_test.go +++ b/backend/app/rest/api/migrator_test.go @@ -34,7 +34,7 @@ func TestMigrator_Import(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} @@ -61,7 +61,7 @@ func TestMigrator_ImportForm(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) bodyBuf := &bytes.Buffer{} @@ -136,7 +136,7 @@ func TestMigrator_ImportRejected(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) client := &http.Client{Timeout: 1 * time.Second} @@ -157,7 +157,7 @@ func TestMigrator_ImportDouble(t *testing.T) { for i := 0; i < 1000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } - r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records + r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) assert.Nil(t, err) @@ -184,7 +184,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) { for i := 0; i < 1000; i++ { recs = append(recs, fmt.Sprintf(tmpl, i)) } - r := strings.NewReader(strings.Join(recs, "\n")) // reader with 10k records + r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records client := &http.Client{Timeout: 1 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r) require.Nil(t, err) @@ -205,7 +205,7 @@ func TestMigrator_Export(t *testing.T) { assert.NotNil(t, srv) defer cleanupImportSrv(srv, ts) - r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} + r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"

test test #1

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"} {"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"

test test #2

","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`) // import comments first @@ -234,7 +234,7 @@ func TestMigrator_Export(t *testing.T) { assert.NoError(t, err) ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) - assert.Equal(t, 2, strings.Count(string(ungzBody), "\n")) + assert.Equal(t, 3, strings.Count(string(ungzBody), "\n")) assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\"")) t.Logf("%s", string(ungzBody)) @@ -248,7 +248,7 @@ func TestMigrator_Export(t *testing.T) { body, err := ioutil.ReadAll(resp.Body) assert.NoError(t, err) - assert.Equal(t, 2, strings.Count(string(body), "\n")) + assert.Equal(t, 3, strings.Count(string(body), "\n")) assert.Equal(t, 2, strings.Count(string(body), "\"text\"")) t.Logf("%s", string(body)) @@ -267,8 +267,8 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http svc = &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, - NativeImporter: &migrator.Remark{DataStore: dataStore}, - NativeExported: &migrator.Remark{DataStore: dataStore}, + NativeImporter: &migrator.Native{DataStore: dataStore}, + NativeExporter: &migrator.Native{DataStore: dataStore}, Cache: &cache.Nop{}, KeyStore: adminStore, } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index 4167a881..050755db 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -142,12 +142,12 @@ func (s *Rest) Shutdown() { s.lock.Unlock() } -func (s *Rest) makeHTTPServer(port int, router chi.Router) *http.Server { +func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server { return &http.Server{ Addr: fmt.Sprintf(":%d", port), Handler: router, ReadHeaderTimeout: 5 * time.Second, - WriteTimeout: 5 * time.Second, + WriteTimeout: 120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request IdleTimeout: 30 * time.Second, } } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index f5600760..d59710a5 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -209,8 +209,8 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { Migrator: &Migrator{ DisqusImporter: &migrator.Disqus{DataStore: dataStore}, WordPressImporter: &migrator.WordPress{DataStore: dataStore}, - NativeImporter: &migrator.Remark{DataStore: dataStore}, - NativeExported: &migrator.Remark{DataStore: dataStore}, + NativeImporter: &migrator.Native{DataStore: dataStore}, + NativeExporter: &migrator.Native{DataStore: dataStore}, Cache: &cache.Nop{}, KeyStore: adminStore, }, diff --git a/backend/app/rest/api/ssl.go b/backend/app/rest/api/ssl.go index a087a613..14a4b728 100644 --- a/backend/app/rest/api/ssl.go +++ b/backend/app/rest/api/ssl.go @@ -82,7 +82,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager { } // makeHTTPSAutoCertServer makes https server with autocert mode (LE support) -func (s *Rest) makeHTTPSAutocertServer(port int, router chi.Router, m *autocert.Manager) *http.Server { +func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server { server := s.makeHTTPServer(port, router) cfg := makeTLSConfig() cfg.GetCertificate = m.GetCertificate @@ -91,7 +91,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router chi.Router, m *autocert. } // makeHTTPSServer makes https server for static mode -func (s *Rest) makeHTTPSServer(port int, router chi.Router) *http.Server { +func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server { server := s.makeHTTPServer(port, router) server.TLSConfig = makeTLSConfig() return server diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go index 1e14c79c..e4ca1384 100644 --- a/backend/app/store/engine/bolt_accessor.go +++ b/backend/app/store/engine/bolt_accessor.go @@ -7,8 +7,8 @@ import ( "strings" "time" - "github.com/coreos/bbolt" - "github.com/hashicorp/go-multierror" + bolt "github.com/coreos/bbolt" + multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go index 1c221d49..5f78b69a 100644 --- a/backend/app/store/engine/bolt_admin.go +++ b/backend/app/store/engine/bolt_admin.go @@ -320,3 +320,20 @@ func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) { }) return verified } + +// Verified returns list of verified userIDs +func (b *BoltDB) Verified(siteID string) (ids []string, err error) { + bdb, err := b.db(siteID) + if err != nil { + return nil, err + } + err = bdb.View(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(verifiedBucketName)) + _ = usersBkt.ForEach(func(k, _ []byte) error { + ids = append(ids, string(k)) + return nil + }) + return nil + }) + return ids, err +} diff --git a/backend/app/store/engine/bolt_admin_test.go b/backend/app/store/engine/bolt_admin_test.go index a46fc4a5..d9bf0ffe 100644 --- a/backend/app/store/engine/bolt_admin_test.go +++ b/backend/app/store/engine/bolt_admin_test.go @@ -214,4 +214,15 @@ func TestBoltAdmin_Verified(t *testing.T) { assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false)) assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") + + assert.NoError(t, b.SetVerified("radio-t", "u1", true)) + assert.NoError(t, b.SetVerified("radio-t", "u2", true)) + assert.NoError(t, b.SetVerified("radio-t", "u3", false)) + + ids, err := b.Verified("radio-t") + assert.NoError(t, err) + assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") + + _, err = b.Verified("radio-t-bad") + assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site") } diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index 8d150808..7355a91c 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -53,6 +53,7 @@ type Admin interface { IsReadOnly(locator store.Locator) bool // check if post read-only SetVerified(siteID string, userID string, status bool) error // set/reset verified flag IsVerified(siteID string, userID string) bool // check verified status + Verified(siteID string) ([]string, error) // list of verified user ids } const ( diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go index a6e50833..587e7bda 100644 --- a/backend/app/store/engine/mongo.go +++ b/backend/app/store/engine/mongo.go @@ -6,7 +6,7 @@ import ( "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/go-pkgz/mongo" - "github.com/hashicorp/go-multierror" + multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" @@ -221,6 +221,21 @@ func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) { return err == nil && meta.Verified } +// Verified returns list of verified user IDs +func (m *Mongo) Verified(siteID string) (ids []string, err error) { + metas := []metaUser{} + err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { + return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas) + }) + if err != nil { + return nil, err + } + for _, m := range metas { + ids = append(ids, m.ID) + } + return ids, nil +} + // SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent // block uses blocksBucketName with key=userID and val=TTL+now func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine/mongo_test.go index 0d94745b..797d2e76 100644 --- a/backend/app/store/engine/mongo_test.go +++ b/backend/app/store/engine/mongo_test.go @@ -239,6 +239,18 @@ func TestMongo_Verified(t *testing.T) { assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false)) assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") + + assert.NoError(t, m.SetVerified("radio-t", "u1", true)) + assert.NoError(t, m.SetVerified("radio-t", "u2", true)) + assert.NoError(t, m.SetVerified("radio-t", "u3", false)) + + ids, err := m.Verified("radio-t") + assert.NoError(t, err) + assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") + + ids, err = m.Verified("radio-t-bad") + assert.NoError(t, err) + assert.Equal(t, 0, len(ids)) } func TestMongo_GetForUser(t *testing.T) { diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 36710160..6a5d94c3 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -1,9 +1,12 @@ package service import ( + "sort" "sync" "time" + multierror "github.com/hashicorp/go-multierror" + "github.com/google/uuid" "github.com/pkg/errors" @@ -28,6 +31,22 @@ type DataStore struct { } } +// UserMetaData keeps info about user flags +type UserMetaData struct { + ID string `json:"id"` + Blocked struct { + Status bool `json:"status"` + Until time.Time `json:"until"` + } `json:"blocked"` + Verified bool `json:"verified"` +} + +// PostMetaData keeps info about post flags +type PostMetaData struct { + URL string `json:"url"` + ReadOnly bool `json:"read_only"` +} + const defaultCommentMaxSize = 2000 // UnlimitedVotes doesn't restrict MaxVotes @@ -206,6 +225,86 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool { return false } +// Metas returns metadata for users and posts +func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) { + umetas = []UserMetaData{} + pmetas = []PostMetaData{} + // set posts meta + posts, err := s.List(siteID, 0, 0) + if err != nil { + return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID) + } + for _, p := range posts { + if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) { + pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true}) + } + + } + + // set users meta + m := map[string]UserMetaData{} + + // process blocked users + blocked, err := s.Blocked(siteID) + if err != nil { + return nil, nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID) + } + for _, b := range blocked { + val, ok := m[b.ID] + if !ok { + val = UserMetaData{ID: b.ID} + } + val.Blocked.Status = true + val.Blocked.Until = b.Until + m[b.ID] = val + } + + // process verified users + verified, err := s.Verified(siteID) + if err != nil { + return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID) + } + for _, v := range verified { + val, ok := m[v] + if !ok { + val = UserMetaData{ID: v} + } + val.Verified = true + m[v] = val + } + + for _, u := range m { + umetas = append(umetas, u) + } + sort.Slice(umetas, func(i, j int) bool { return umetas[i].ID < umetas[j].ID }) + + return umetas, pmetas, nil +} + +// SetMetas saves metadata for users and posts +func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []PostMetaData) (err error) { + errs := new(multierror.Error) + + // save posts metas + for _, pm := range pmetas { + if pm.ReadOnly { + errs = multierror.Append(errs, s.SetReadOnly(store.Locator{SiteID: siteID, URL: pm.URL}, true)) + } + } + + // save users metas + for _, um := range umetas { + if um.Blocked.Status { + errs = multierror.Append(errs, s.SetBlock(siteID, um.ID, true, time.Until(um.Blocked.Until))) + } + if um.Verified { + errs = multierror.Append(errs, s.SetVerified(siteID, um.ID, true)) + } + } + + return errs.ErrorOrNil() +} + // getsScopedLocks pull lock from the map if found or create a new one func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) { s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} }) diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index b5b22297..257d5891 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -374,6 +374,57 @@ func TestService_Counts(t *testing.T) { }, res) } +func TestService_GetMetas(t *testing.T) { + defer os.Remove(testDb) + // two comments for https://radio-t.com + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticKeyStore("secret 123")} + + um, pm, err := b.Metas("radio-t") + require.NoError(t, err) + assert.Equal(t, 0, len(um)) + assert.Equal(t, 0, len(pm)) + + assert.NoError(t, b.SetVerified("radio-t", "user1", true)) + assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true)) + + um, pm, err = b.Metas("radio-t") + require.NoError(t, err) + + assert.Equal(t, 1, len(um)) + assert.Equal(t, "user1", um[0].ID) + assert.Equal(t, true, um[0].Verified) + assert.Equal(t, false, um[0].Blocked.Status) + + assert.Equal(t, 1, len(pm)) + assert.Equal(t, "https://radio-t.com", pm[0].URL) + assert.Equal(t, true, pm[0].ReadOnly) +} + +func TestService_SetMetas(t *testing.T) { + defer os.Remove(testDb) + // two comments for https://radio-t.com + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticKeyStore("secret 123")} + umetas := []UserMetaData{} + pmetas := []PostMetaData{} + err := b.SetMetas("radio-t", umetas, pmetas) + assert.NoError(t, err, "empty metas") + + um1 := UserMetaData{ID: "user1", Verified: true} + um2 := UserMetaData{ID: "user2"} + um2.Blocked.Status = true + um2.Blocked.Until = time.Now().AddDate(0, 1, 1) + + pmetas = []PostMetaData{{URL: "https://radio-t.com", ReadOnly: true}} + err = b.SetMetas("radio-t", []UserMetaData{um1, um2}, pmetas) + assert.NoError(t, err) + + assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"})) + assert.True(t, b.IsVerified("radio-t", "user1")) + assert.True(t, b.IsBlocked("radio-t", "user2")) +} + // makes new boltdb, put two records func prepStoreEngine(t *testing.T) engine.Interface { os.Remove(testDb)