From a691210fd6463e33c9441b3d25996867dd04d711 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 2 May 2018 23:13:50 -0500 Subject: [PATCH] simplify interfaces for migrator, switch back to dataStore --- app/main.go | 6 +++--- app/migrator/disqus.go | 6 +++--- app/migrator/disqus_test.go | 2 +- app/migrator/migrator.go | 18 +++--------------- app/migrator/migrator_test.go | 16 ++++++++-------- app/migrator/remark.go | 13 ++++++------- app/migrator/remark_test.go | 4 ++-- app/rest/api/import_test.go | 4 ++-- app/rest/api/rest_test.go | 7 +++---- app/store/bolt.go | 10 +++++++--- 10 files changed, 38 insertions(+), 48 deletions(-) diff --git a/app/main.go b/app/main.go index 40db0c83..54e3f25a 100644 --- a/app/main.go +++ b/app/main.go @@ -82,7 +82,7 @@ func main() { return sess }() - exporter := migrator.Remark{CommentFinder: &dataService} + exporter := migrator.Remark{DataStore: &dataService} cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn) activateBackup(&exporter) @@ -90,8 +90,8 @@ func main() { importSrv := api.Import{ Version: revision, Cache: cache, - NativeImporter: &migrator.Remark{CommentCreator: &dataService}, - DisqusImporter: &migrator.Disqus{CommentCreator: &dataService}, + NativeImporter: &migrator.Remark{DataStore: &dataService}, + DisqusImporter: &migrator.Disqus{DataStore: &dataService}, } go importSrv.Run(opts.Port + 1) diff --git a/app/migrator/disqus.go b/app/migrator/disqus.go index d030c59a..301616f3 100644 --- a/app/migrator/disqus.go +++ b/app/migrator/disqus.go @@ -14,7 +14,7 @@ import ( // Disqus implements Importer from disqus xml type Disqus struct { - CommentCreator + DataStore store.Interface } type disqusThread struct { @@ -53,14 +53,14 @@ type uid struct { // Import from disqus and save to store func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) { - if err = d.DeleteAll(siteID); err != nil { + if err = d.DataStore.DeleteAll(siteID); err != nil { return 0, err } commentsCh := d.convert(r, siteID) failed, passed := 0, 0 for c := range commentsCh { - if _, err = d.Create(c); err != nil { + if _, err = d.DataStore.Create(c); err != nil { failed++ continue } diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index b71c7f60..8177963d 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -17,7 +17,7 @@ func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - d := Disqus{CommentCreator: dataStore} + d := Disqus{DataStore: dataStore} size, err := d.Import(strings.NewReader(xmlTest), "test") assert.Nil(t, err) assert.Equal(t, 3, size) diff --git a/app/migrator/migrator.go b/app/migrator/migrator.go index b1912b7d..863f82d2 100644 --- a/app/migrator/migrator.go +++ b/app/migrator/migrator.go @@ -23,21 +23,9 @@ type Exporter interface { Export(w io.Writer, siteID string) (int, error) } -// CommentCreator is a minimal interface used by importer to make comments -type CommentCreator interface { - Create(comment store.Comment) (commentID string, err error) - DeleteAll(siteID string) error -} - -// CommentFinder is a minimal interface used by exporter to find comments and list posts -type CommentFinder interface { - Find(locator store.Locator, sort string) ([]store.Comment, error) - List(siteID string, limit int, skip int) ([]store.PostInfo, error) -} - // ImportParams defines everything needed to run import type ImportParams struct { - CommentCreator + DataStore store.Interface InputFile string Provider string SiteID string @@ -50,9 +38,9 @@ func ImportComments(p ImportParams) (int, error) { var importer Importer switch p.Provider { case "disqus": - importer = &Disqus{CommentCreator: p.CommentCreator} + importer = &Disqus{DataStore: p.DataStore} case "native": - importer = &Remark{CommentCreator: p.CommentCreator} + importer = &Remark{DataStore: p.DataStore} default: return 0, errors.Errorf("unsupported import provider %s", p.Provider) } diff --git a/app/migrator/migrator_test.go b/app/migrator/migrator_test.go index 6a38ea1b..fbec563b 100644 --- a/app/migrator/migrator_test.go +++ b/app/migrator/migrator_test.go @@ -25,10 +25,10 @@ func TestMigrator_ImportDisqus(t *testing.T) { require.Nil(t, err, "create store") size, err := ImportComments(ImportParams{ - CommentCreator: dataStore, - InputFile: "/tmp/disqus-test.xml", - SiteID: "test", - Provider: "disqus", + DataStore: dataStore, + InputFile: "/tmp/disqus-test.xml", + SiteID: "test", + Provider: "disqus", }) assert.Nil(t, err) assert.Equal(t, 3, size) @@ -54,10 +54,10 @@ func TestMigrator_ImportRemark(t *testing.T) { require.Nil(t, err, "create store") size, err := ImportComments(ImportParams{ - CommentCreator: dataStore, - InputFile: "/tmp/disqus-test.r42", - SiteID: "radio-t", - Provider: "native", + DataStore: dataStore, + InputFile: "/tmp/disqus-test.r42", + SiteID: "radio-t", + Provider: "native", }) assert.Nil(t, err) assert.Equal(t, 2, size) diff --git a/app/migrator/remark.go b/app/migrator/remark.go index 823778db..f7d98b8d 100644 --- a/app/migrator/remark.go +++ b/app/migrator/remark.go @@ -12,15 +12,14 @@ import ( "github.com/umputun/remark/app/store" ) -// Remark implements exporter and importer for internal store +// Remark implements exporter and importer for internal store format type Remark struct { - CommentCreator - CommentFinder + DataStore store.Interface } // Export all comments to writer as json strings. Each comment is one string, separated by "\n" func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) { - topics, err := r.List(siteID, 0, 0) + topics, err := r.DataStore.List(siteID, 0, 0) if err != nil { return 0, err } @@ -29,7 +28,7 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) { commentsCount := 0 for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction topic := topics[i] - comments, err := r.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") + comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") if err != nil { return commentsCount, err } @@ -57,7 +56,7 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) { // Import comments from json strings produced by Remark.Export func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) { - if err := r.DeleteAll(siteID); err != nil { + if err := r.DataStore.DeleteAll(siteID); err != nil { return 0, err } @@ -76,7 +75,7 @@ func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) { log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err) continue } - if _, err := r.Create(comment); err != nil { + if _, err := r.DataStore.Create(comment); err != nil { failed++ log.Printf("[WARN] can't write %+v to store, %s", comment, err) continue diff --git a/app/migrator/remark_test.go b/app/migrator/remark_test.go index cc50c4fb..f46b2f01 100644 --- a/app/migrator/remark_test.go +++ b/app/migrator/remark_test.go @@ -17,7 +17,7 @@ var testDb = "/tmp/test-remark.db" func TestRemark_Export(t *testing.T) { b := prep(t) - r := Remark{CommentFinder: b} + r := Remark{DataStore: b} buf := &bytes.Buffer{} size, err := r.Export(buf, "radio-t") @@ -43,7 +43,7 @@ func TestRemark_Import(t *testing.T) { os.Remove(testDb) b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - r := Remark{CommentCreator: b} + r := Remark{DataStore: b} size, err := r.Import(buf, "radio-t") assert.Nil(t, err) assert.Equal(t, 2, size) diff --git a/app/rest/api/import_test.go b/app/rest/api/import_test.go index 2d7f4e8b..d5045eb9 100644 --- a/app/rest/api/import_test.go +++ b/app/rest/api/import_test.go @@ -37,8 +37,8 @@ func prepImportSrv(t *testing.T) (srv *Import, port int) { require.Nil(t, err) srv = &Import{ - DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, - NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + DisqusImporter: &migrator.Disqus{DataStore: dataStore}, + NativeImporter: &migrator.Remark{DataStore: dataStore}, Cache: &mockCache{}, } diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index 29ad7969..9dd78e9a 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -23,7 +23,6 @@ import ( "github.com/umputun/remark/app/store" ) - var testDb = "/tmp/test-remark.db" var testHTML = "/tmp/test-remark.html" @@ -422,14 +421,14 @@ func prep(t *testing.T) (srv *Rest, port int) { AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"}, Admins: []string{"a1", "a2"}, }, - Exporter: &migrator.Remark{CommentFinder: dataStore}, + Exporter: &migrator.Remark{DataStore: dataStore}, Cache: &mockCache{}, WebRoot: "/tmp", } importSrv := &Import{ - DisqusImporter: &migrator.Disqus{CommentCreator: dataStore}, - NativeImporter: &migrator.Remark{CommentCreator: dataStore}, + DisqusImporter: &migrator.Disqus{DataStore: dataStore}, + NativeImporter: &migrator.Remark{DataStore: dataStore}, Cache: &mockCache{}, } diff --git a/app/store/bolt.go b/app/store/bolt.go index fa9cc300..2bbacd50 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -38,8 +38,6 @@ const ( userLimit = 50 ) -var topBuckets = []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName} - const tsNano = "2006-01-02T15:04:05.000000000Z07:00" // BoltSite defines single site param @@ -60,6 +58,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { } // make top-level buckets + topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName} err = db.Update(func(tx *bolt.Tx) error { for _, bktName := range topBuckets { if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { @@ -183,9 +182,14 @@ func (b *BoltDB) DeleteAll(siteID string) error { if err != nil { return err } + + // delete all buckets except blocked users + toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName} + // delete top-level buckets err = bdb.Update(func(tx *bolt.Tx) error { - for _, bktName := range topBuckets { + for _, bktName := range toDelete { + if e := tx.DeleteBucket([]byte(bktName)); e != nil { return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) }