mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify interfaces for migrator, switch back to dataStore

Browse Source
This commit is contained in:
Umputun
2018-05-02 23:13:50 -05:00
parent 797bb1b5ed
commit a691210fd6
10 changed files with 38 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.go
+3 -3
View File
@@ -82,7 +82,7 @@ func main() {
return sess
}()
exporter := migrator.Remark{CommentFinder: &dataService}
exporter := migrator.Remark{DataStore: &dataService}
cache := rest.NewLoadingCache(4*time.Hour, 15*time.Minute, postFlushFn)
activateBackup(&exporter)
@@ -90,8 +90,8 @@ func main() {
importSrv := api.Import{
Version: revision,
Cache: cache,
NativeImporter: &migrator.Remark{CommentCreator: &dataService},
DisqusImporter: &migrator.Disqus{CommentCreator: &dataService},
NativeImporter: &migrator.Remark{DataStore: &dataService},
DisqusImporter: &migrator.Disqus{DataStore: &dataService},
}
go importSrv.Run(opts.Port + 1)
app/migrator/disqus.go
+3 -3
View File
@@ -14,7 +14,7 @@ import (
// Disqus implements Importer from disqus xml
type Disqus struct {
CommentCreator
DataStore store.Interface
}
type disqusThread struct {
@@ -53,14 +53,14 @@ type uid struct {
// Import from disqus and save to store
func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
if err = d.DeleteAll(siteID); err != nil {
if err = d.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
commentsCh := d.convert(r, siteID)
failed, passed := 0, 0
for c := range commentsCh {
if _, err = d.Create(c); err != nil {
if _, err = d.DataStore.Create(c); err != nil {
failed++
continue
}
app/migrator/disqus_test.go
+1 -1
View File
@@ -17,7 +17,7 @@ func TestDisqus_Import(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
dataStore, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
d := Disqus{CommentCreator: dataStore}
d := Disqus{DataStore: dataStore}
size, err := d.Import(strings.NewReader(xmlTest), "test")
assert.Nil(t, err)
assert.Equal(t, 3, size)
app/migrator/migrator.go
+3 -15
View File
@@ -23,21 +23,9 @@ type Exporter interface {
Export(w io.Writer, siteID string) (int, error)
}
// CommentCreator is a minimal interface used by importer to make comments
type CommentCreator interface {
Create(comment store.Comment) (commentID string, err error)
DeleteAll(siteID string) error
}
// CommentFinder is a minimal interface used by exporter to find comments and list posts
type CommentFinder interface {
Find(locator store.Locator, sort string) ([]store.Comment, error)
List(siteID string, limit int, skip int) ([]store.PostInfo, error)
}
// ImportParams defines everything needed to run import
type ImportParams struct {
CommentCreator
DataStore store.Interface
InputFile string
Provider string
SiteID string
@@ -50,9 +38,9 @@ func ImportComments(p ImportParams) (int, error) {
var importer Importer
switch p.Provider {
case "disqus":
importer = &Disqus{CommentCreator: p.CommentCreator}
importer = &Disqus{DataStore: p.DataStore}
case "native":
importer = &Remark{CommentCreator: p.CommentCreator}
importer = &Remark{DataStore: p.DataStore}
default:
return 0, errors.Errorf("unsupported import provider %s", p.Provider)
}
app/migrator/migrator_test.go
+8 -8
View File
@@ -25,10 +25,10 @@ func TestMigrator_ImportDisqus(t *testing.T) {
require.Nil(t, err, "create store")
size, err := ImportComments(ImportParams{
CommentCreator: dataStore,
InputFile: "/tmp/disqus-test.xml",
SiteID: "test",
Provider: "disqus",
DataStore: dataStore,
InputFile: "/tmp/disqus-test.xml",
SiteID: "test",
Provider: "disqus",
})
assert.Nil(t, err)
assert.Equal(t, 3, size)
@@ -54,10 +54,10 @@ func TestMigrator_ImportRemark(t *testing.T) {
require.Nil(t, err, "create store")
size, err := ImportComments(ImportParams{
CommentCreator: dataStore,
InputFile: "/tmp/disqus-test.r42",
SiteID: "radio-t",
Provider: "native",
DataStore: dataStore,
InputFile: "/tmp/disqus-test.r42",
SiteID: "radio-t",
Provider: "native",
})
assert.Nil(t, err)
assert.Equal(t, 2, size)
app/migrator/remark.go
+6 -7
View File
@@ -12,15 +12,14 @@ import (
"github.com/umputun/remark/app/store"
)
// Remark implements exporter and importer for internal store
// Remark implements exporter and importer for internal store format
type Remark struct {
CommentCreator
CommentFinder
DataStore store.Interface
}
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
topics, err := r.List(siteID, 0, 0)
topics, err := r.DataStore.List(siteID, 0, 0)
if err != nil {
return 0, err
}
@@ -29,7 +28,7 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
commentsCount := 0
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
topic := topics[i]
comments, err := r.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
if err != nil {
return commentsCount, err
}
@@ -57,7 +56,7 @@ func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
// Import comments from json strings produced by Remark.Export
func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
if err := r.DeleteAll(siteID); err != nil {
if err := r.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
@@ -76,7 +75,7 @@ func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
continue
}
if _, err := r.Create(comment); err != nil {
if _, err := r.DataStore.Create(comment); err != nil {
failed++
log.Printf("[WARN] can't write %+v to store, %s", comment, err)
continue
app/migrator/remark_test.go
+2 -2
View File
@@ -17,7 +17,7 @@ var testDb = "/tmp/test-remark.db"
func TestRemark_Export(t *testing.T) {
b := prep(t)
r := Remark{CommentFinder: b}
r := Remark{DataStore: b}
buf := &bytes.Buffer{}
size, err := r.Export(buf, "radio-t")
@@ -43,7 +43,7 @@ func TestRemark_Import(t *testing.T) {
os.Remove(testDb)
b, err := store.NewBoltDB(bolt.Options{}, store.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{CommentCreator: b}
r := Remark{DataStore: b}
size, err := r.Import(buf, "radio-t")
assert.Nil(t, err)
assert.Equal(t, 2, size)
app/rest/api/import_test.go
+2 -2
View File
@@ -37,8 +37,8 @@ func prepImportSrv(t *testing.T) (srv *Import, port int) {
require.Nil(t, err)
srv = &Import{
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
Cache: &mockCache{},
}
app/rest/api/rest_test.go
+3 -4
View File
@@ -23,7 +23,6 @@ import (
"github.com/umputun/remark/app/store"
)
var testDb = "/tmp/test-remark.db"
var testHTML = "/tmp/test-remark.html"
@@ -422,14 +421,14 @@ func prep(t *testing.T) (srv *Rest, port int) {
AvatarProxy: &auth.AvatarProxy{StorePath: "/tmp", RoutePath: "/api/v1/avatar"},
Admins: []string{"a1", "a2"},
},
Exporter: &migrator.Remark{CommentFinder: dataStore},
Exporter: &migrator.Remark{DataStore: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
}
importSrv := &Import{
DisqusImporter: &migrator.Disqus{CommentCreator: dataStore},
NativeImporter: &migrator.Remark{CommentCreator: dataStore},
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
Cache: &mockCache{},
}
app/store/bolt.go
+7 -3
View File
@@ -38,8 +38,6 @@ const (
userLimit = 50
)
var topBuckets = []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
// BoltSite defines single site param
@@ -60,6 +58,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
}
// make top-level buckets
topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, countsBucketName}
err = db.Update(func(tx *bolt.Tx) error {
for _, bktName := range topBuckets {
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
@@ -183,9 +182,14 @@ func (b *BoltDB) DeleteAll(siteID string) error {
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range topBuckets {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}