diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 0213e388..fdb2fc24 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -31,7 +31,7 @@ import (
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
- "github.com/umputun/remark/backend/app/store/engine"
+ "github.com/umputun/remark/backend/app/store/engine2"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -249,7 +249,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
dataService := &service.DataStore{
- Interface: storeEngine,
+ Engine: storeEngine,
EditDuration: s.EditDuration,
AdminStore: adminStore,
MaxCommentSize: s.MaxCommentSize,
@@ -401,7 +401,7 @@ func (a *serverApp) activateBackup(ctx context.Context) {
}
// makeDataStore creates store for all sites
-func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
+func (s *ServerCommand) makeDataStore() (result engine2.Interface, err error) {
log.Printf("[INFO] make data store, type=%s", s.Store.Type)
switch s.Store.Type {
@@ -409,18 +409,18 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
if err = makeDirs(s.Store.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
- sites := []engine.BoltSite{}
+ sites := []engine2.BoltSite{}
for _, site := range s.Sites {
- sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
+ sites = append(sites, engine2.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
}
- result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
- case "mongo":
- mgServer, e := s.makeMongo()
- if e != nil {
- return result, errors.Wrap(e, "failed to create mongo server")
- }
- conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
- result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+ result, err = engine2.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
+ // case "mongo":
+ // mgServer, e := s.makeMongo()
+ // if e != nil {
+ // return result, errors.Wrap(e, "failed to create mongo server")
+ // }
+ // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
+ // result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
}
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index a0e8566a..d4fedc31 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -31,7 +31,7 @@ import (
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
- "github.com/umputun/remark/backend/app/store/engine"
+ "github.com/umputun/remark/backend/app/store/engine2"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -291,7 +291,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
os.RemoveAll("/tmp/ava-remark42")
os.RemoveAll("/tmp/pics-remark42")
- b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
+ b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
memCache, err := cache.NewMemoryCache()
@@ -301,7 +301,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
dataStore := &service.DataStore{
- Interface: b,
+ Engine: b,
EditDuration: 5 * time.Minute,
MaxCommentSize: 4000,
AdminStore: adminStore,
diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 1a555937..c900d774 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -105,7 +105,7 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
}
// serialize comment to json []byte for bolt and save
- if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
+ if e = b.save(postBkt, comment.ID, comment); e != nil {
return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
}
@@ -206,7 +206,7 @@ func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store
}
comment := store.Comment{}
- if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
+ if e = b.load(postBkt, commentID, &comment); e != nil {
log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
continue
}
@@ -263,7 +263,7 @@ func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err
postURL := string(k)
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
- if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
+ if e := b.load(infoBkt, postURL, &info); e != nil {
return errors.Wrapf(e, "can't load info for %s", postURL)
}
list = append(list, info)
@@ -287,7 +287,7 @@ func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, e
info := store.PostInfo{}
err = bdb.View(func(tx *bolt.Tx) error {
infoBkt := tx.Bucket([]byte(infoBucketName))
- if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
+ if e := b.load(infoBkt, locator.URL, &info); e != nil {
return errors.Wrapf(e, "can't load info for %s", locator.URL)
}
return nil
@@ -391,7 +391,7 @@ func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Com
if e != nil {
return e
}
- return b.load(bucket, []byte(commentID), &comment)
+ return b.load(bucket, commentID, &comment)
})
return comment, err
}
@@ -417,7 +417,7 @@ func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
if e != nil {
return e
}
- return b.save(bucket, []byte(comment.ID), comment)
+ return b.save(bucket, comment.ID, comment)
})
}
@@ -467,7 +467,7 @@ func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error)
}
// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
+func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
if value == nil {
return errors.Errorf("can't save nil value for %s", key)
}
@@ -475,15 +475,15 @@ func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err erro
if jerr != nil {
return errors.Wrap(jerr, "can't marshal comment")
}
- if err = bkt.Put(key, jdata); err != nil {
+ if err = bkt.Put([]byte(key), jdata); err != nil {
return errors.Wrapf(err, "failed to save key %s", key)
}
return nil
}
// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
- value := bkt.Get(key)
+func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
+ value := bkt.Get([]byte(key))
if value == nil {
return errors.Errorf("no value for %s", key)
}
@@ -501,7 +501,7 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
- if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
+ if err := b.load(infoBkt, postURL, &info); err != nil {
info = store.PostInfo{}
}
if val == 0 { // get current count, don't update
@@ -509,13 +509,13 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
}
info.Count += val
- return info.Count, b.save(infoBkt, []byte(postURL), &info)
+ return info.Count, b.save(infoBkt, postURL, &info)
}
func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
- if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
+ if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
info = store.PostInfo{
Count: 0,
URL: comment.Locator.URL,
@@ -525,7 +525,7 @@ func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, er
}
info.Count++
info.LastTS = comment.Timestamp
- return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
+ return info, b.save(infoBkt, comment.Locator.URL, &info)
}
func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go
index 429ac991..8924f560 100644
--- a/backend/app/store/engine/bolt_admin.go
+++ b/backend/app/store/engine/bolt_admin.go
@@ -29,13 +29,13 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
}
comment := store.Comment{}
- if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
+ if err = b.load(postBkt, commentID, &comment); err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted(mode)
- if err = b.save(postBkt, []byte(commentID), comment); err != nil {
+ if err = b.save(postBkt, commentID, comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go
index fb6b2513..5c6a6827 100644
--- a/backend/app/store/engine/engine.go
+++ b/backend/app/store/engine/engine.go
@@ -13,14 +13,6 @@ import (
// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
-// UserRequest is the request send to get comments by user
-type UserRequest struct {
- SiteID string
- UserID string
- Limit int
- Skip int
-}
-
// Interface defines methods provided by low-level storage engine
type Interface interface {
Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
diff --git a/backend/app/store/engine2/bolt.go b/backend/app/store/engine2/bolt.go
new file mode 100644
index 00000000..2a5c6937
--- /dev/null
+++ b/backend/app/store/engine2/bolt.go
@@ -0,0 +1,863 @@
+package engine2
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "log"
+ "strings"
+ "time"
+
+ bolt "github.com/coreos/bbolt"
+ "github.com/hashicorp/go-multierror"
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
+// there are 5 types of top-level buckets:
+// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
+// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
+// value is not full comment but a reference combined from post-url+commentID
+// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
+// is a nested bucket named userID with kv as ts:reference
+// - blocking info sits in "block" bucket. Key is userID, value - ts
+// - counts per post to keep number of comments. Key is post url, value - count
+// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
+type BoltDB struct {
+ dbs map[string]*bolt.DB
+}
+
+const (
+ // top level buckets
+ postsBucketName = "posts"
+ lastBucketName = "last"
+ userBucketName = "users"
+ blocksBucketName = "block"
+ infoBucketName = "info"
+ readonlyBucketName = "readonly"
+ verifiedBucketName = "verified"
+
+ tsNano = "2006-01-02T15:04:05.000000000Z07:00"
+)
+
+// BoltSite defines single site param
+type BoltSite struct {
+ FileName string // full path to boltdb
+ SiteID string // ID of given site
+}
+
+// NewBoltDB makes persistent boltdb-based store. For each site new boltdb file created
+func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
+ log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
+ result := BoltDB{dbs: make(map[string]*bolt.DB)}
+ for _, site := range sites {
+ db, err := bolt.Open(site.FileName, 0600, &options)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
+ }
+
+ // make top-level buckets
+ topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
+ readonlyBucketName, verifiedBucketName}
+ err = db.Update(func(tx *bolt.Tx) error {
+ for _, bktName := range topBuckets {
+ if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+ }
+ }
+ return nil
+ })
+
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create top level bucket)")
+ }
+
+ result.dbs[site.SiteID] = db
+ log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
+ }
+ return &result, nil
+}
+
+// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
+func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
+
+ bdb, err := b.db(comment.Locator.SiteID)
+ if err != nil {
+ return "", err
+ }
+
+ if b.checkFlag(FlagRequest{Locator: comment.Locator, Flag: ReadOnly}) {
+ return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+ }
+
+ err = bdb.Update(func(tx *bolt.Tx) (err error) {
+ var postBkt, lastBkt, userBkt *bolt.Bucket
+
+ if postBkt, err = b.makePostBucket(tx, comment.Locator.URL); err != nil {
+ return err
+ }
+ // check if key already in store, reject doubles
+ if postBkt.Get([]byte(comment.ID)) != nil {
+ return errors.Errorf("key %s already in store", comment.ID)
+ }
+
+ // serialize comment to json []byte for bolt and save
+ if err = b.save(postBkt, comment.ID, comment); err != nil {
+ return errors.Wrapf(err, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
+ }
+
+ ref := b.makeRef(comment) // reference combines url and comment id
+
+ // add reference to comment to "last" bucket
+ lastBkt = tx.Bucket([]byte(lastBucketName))
+ commentTs := []byte(comment.Timestamp.Format(tsNano))
+ if err = lastBkt.Put(commentTs, ref); err != nil {
+ return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
+ }
+
+ // add reference to commentID to "users" bucket
+ if userBkt, err = b.getUserBucket(tx, comment.User.ID); err != nil {
+ return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
+ }
+ // put into individual user's bucket with ts as a key
+ if err = userBkt.Put(commentTs, ref); err != nil {
+ return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
+ }
+
+ // set info with the count for post url
+ if _, err = b.setInfo(tx, comment); err != nil {
+ return errors.Wrapf(err, "failed to set info for %s", comment.Locator)
+ }
+ return nil
+ })
+
+ return comment.ID, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
+
+ bdb, err := b.db(locator.SiteID)
+ if err != nil {
+ return comment, err
+ }
+
+ err = bdb.View(func(tx *bolt.Tx) error {
+ bucket, e := b.getPostBucket(tx, locator.URL)
+ if e != nil {
+ return e
+ }
+ return b.load(bucket, commentID, &comment)
+ })
+ return comment, err
+}
+
+// Find returns all comments for given request and sorts results
+func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
+ comments = []store.Comment{}
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return nil, err
+ }
+
+ switch {
+ case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+ err = bdb.View(func(tx *bolt.Tx) error {
+
+ bucket, e := b.getPostBucket(tx, req.Locator.URL)
+ if e != nil {
+ return e
+ }
+
+ return bucket.ForEach(func(k, v []byte) error {
+ comment := store.Comment{}
+ if e = json.Unmarshal(v, &comment); e != nil {
+ return errors.Wrap(e, "failed to unmarshal")
+ }
+ comments = append(comments, comment)
+ return nil
+ })
+ })
+ case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+ comments, err = b.lastComments(req.Locator.SiteID, req.Limit, req.Since)
+ case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+ comments, err = b.userComments(req.Locator.SiteID, req.UserID, req.Limit, req.Skip)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ return SortComments(comments, req.Sort), nil
+}
+
+// Flag sets and gets flag values
+func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
+ if req.Update == FlagNonSet { // read flag value, no update requested
+ return b.checkFlag(req), nil
+ }
+
+ // write flag value
+ return b.setFlag(req)
+}
+
+// Update for locator.URL with mutable part of comment
+func (b *BoltDB) Update(locator store.Locator, comment store.Comment) error {
+
+ if curComment, err := b.Get(locator, comment.ID); err == nil {
+ // preserve immutable fields
+ comment.ParentID = curComment.ParentID
+ comment.Locator = curComment.Locator
+ comment.Timestamp = curComment.Timestamp
+ comment.User = curComment.User
+ }
+
+ bdb, err := b.db(locator.SiteID)
+ if err != nil {
+ return err
+ }
+
+ return bdb.Update(func(tx *bolt.Tx) error {
+ bucket, e := b.getPostBucket(tx, locator.URL)
+ if e != nil {
+ return e
+ }
+ return b.save(bucket, comment.ID, comment)
+ })
+}
+
+// Count returns number of comments for post or user
+func (b *BoltDB) Count(req FindRequest) (count int, err error) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return 0, err
+ }
+
+ if req.Locator.URL != "" { // comment's count for post
+ err = bdb.View(func(tx *bolt.Tx) error {
+ var e error
+ count, e = b.count(tx, req.Locator.URL, 0)
+ return e
+ })
+ return count, err
+ }
+
+ if req.UserID != "" { // comment's count for user
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt := usersBkt.Bucket([]byte(req.UserID))
+ if userIDBkt == nil {
+ return errors.Errorf("no comments for user %s in store for %s site", req.UserID, req.Locator.SiteID)
+ }
+ stats := userIDBkt.Stats()
+ count = stats.KeyN
+ return nil
+ })
+ return count, err
+ }
+
+ return 0, errors.Errorf("invalid count request %+v", req)
+}
+
+// Info get post(s) meta info
+func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return []store.PostInfo{}, err
+ }
+
+ if req.Locator.URL != "" { // post info
+ info := store.PostInfo{}
+ err = bdb.View(func(tx *bolt.Tx) error {
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ if e := b.load(infoBkt, req.Locator.URL, &info); e != nil {
+ return errors.Wrapf(e, "can't load info for %s", req.Locator.URL)
+ }
+ return nil
+ })
+
+ // set read-only from age and manual bucket
+ readOnlyAge := req.ReadOnlyAge
+ info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+ if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+ info.ReadOnly = true
+ }
+ return []store.PostInfo{info}, err
+ }
+
+ if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+ list := []store.PostInfo{}
+ err = bdb.View(func(tx *bolt.Tx) error {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+
+ c := postsBkt.Cursor()
+ n := 0
+ for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
+ n++
+ if req.Skip > 0 && n <= req.Skip {
+ continue
+ }
+ postURL := string(k)
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ info := store.PostInfo{}
+ if e := b.load(infoBkt, postURL, &info); e != nil {
+ return errors.Wrapf(e, "can't load info for %s", postURL)
+ }
+ list = append(list, info)
+ if req.Limit > 0 && len(list) >= req.Limit {
+ break
+ }
+ }
+ return nil
+ })
+ return list, err
+ }
+
+ return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (b *BoltDB) ListFlags(siteID string, flag Flag) (res []interface{}, err error) {
+
+ bdb, e := b.db(siteID)
+ if e != nil {
+ return nil, e
+ }
+
+ switch flag {
+ case Verified:
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(verifiedBucketName))
+ _ = usersBkt.ForEach(func(k, _ []byte) error {
+ res = append(res, string(k))
+ return nil
+ })
+ return nil
+ })
+ return res, err
+ case Blocked:
+ err = bdb.View(func(tx *bolt.Tx) error {
+ bucket := tx.Bucket([]byte(blocksBucketName))
+ return bucket.ForEach(func(k []byte, v []byte) error {
+ ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+ if errParse != nil {
+ return errors.Wrap(errParse, "can't parse block ts")
+ }
+ if time.Now().Before(ts) {
+ // get user name from comment user section
+ userName := ""
+ req := FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: string(k), Limit: 1}
+ userComments, errUser := b.Find(req)
+ if errUser == nil && len(userComments) > 0 {
+ userName = userComments[0].User.Name
+ }
+ res = append(res, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
+ }
+ return nil
+ })
+ })
+ return res, err
+ }
+ return nil, errors.Errorf("flag %s not listable", flag)
+}
+
+// Delete post(s) by id or by userID
+func (b *BoltDB) Delete(req DeleteRequest) error {
+
+ bdb, e := b.db(req.Locator.SiteID)
+ if e != nil {
+ return e
+ }
+
+ switch {
+ case req.Locator.URL != "" && req.CommentID != "":
+ return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
+ case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
+ return b.deleteUser(bdb, req.Locator.SiteID, req.UserID)
+ case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
+ return b.deleteAll(bdb, req.Locator.SiteID)
+ }
+
+ return errors.Errorf("invalid delete request %+v", req)
+}
+
+// Close boltdb store
+func (b *BoltDB) Close() error {
+ errs := new(multierror.Error)
+ for site, db := range b.dbs {
+ err := errors.Wrapf(db.Close(), "can't close site %s", site)
+ errs = multierror.Append(errs, err)
+ }
+ return errs.ErrorOrNil()
+}
+
+// Last returns up to max last comments for given siteID
+func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+
+ comments = []store.Comment{}
+
+ if max > lastLimit || max == 0 {
+ max = lastLimit
+ }
+
+ bdb, err := b.db(siteID)
+ if err != nil {
+ return nil, err
+ }
+
+ err = bdb.View(func(tx *bolt.Tx) error {
+ lastBkt := tx.Bucket([]byte(lastBucketName))
+ c := lastBkt.Cursor()
+
+ for k, v := c.Last(); k != nil; k, v = c.Prev() {
+
+ if !since.IsZero() {
+ // stop if reached "since" ts
+ tsSince := []byte(since.Format(tsNano))
+ if bytes.Compare(k, tsSince) <= 0 {
+ break
+ }
+ }
+ url, commentID, e := b.parseRef(v)
+ if e != nil {
+ return e
+ }
+ postBkt, e := b.getPostBucket(tx, url)
+ if e != nil {
+ return e
+ }
+
+ comment := store.Comment{}
+ if e = b.load(postBkt, commentID, &comment); e != nil {
+ log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
+ continue
+ }
+ if comment.Deleted {
+ continue
+ }
+ comments = append(comments, comment)
+ if len(comments) >= max {
+ break
+ }
+ }
+ return nil
+ })
+
+ return comments, err
+}
+
+// userComments extracts all comments for given site and given userID
+// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
+func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+
+ comments = []store.Comment{}
+ commentRefs := []string{}
+
+ bdb, err := b.db(siteID)
+ if err != nil {
+ return nil, err
+ }
+
+ if limit == 0 || limit > userLimit {
+ limit = userLimit
+ }
+
+ // get list of references to comments
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt := usersBkt.Bucket([]byte(userID))
+ if userIDBkt == nil {
+ return errors.Errorf("no comments for user %s in store", userID)
+ }
+
+ c := userIDBkt.Cursor()
+ skipComments := 0
+ for k, v := c.Last(); k != nil; k, v = c.Prev() {
+ if len(commentRefs) >= limit {
+ break
+ }
+ if skip > 0 && skipComments < skip {
+ skipComments++
+ continue
+ }
+ commentRefs = append(commentRefs, string(v))
+ }
+ return nil
+ })
+
+ if err != nil {
+ return comments, err
+ }
+
+ // retrieve comments for refs
+ for _, v := range commentRefs {
+ url, commentID, errParse := b.parseRef([]byte(v))
+ if errParse != nil {
+ return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
+ }
+ if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
+ comments = append(comments, c)
+ }
+ }
+
+ return comments, err
+}
+
+func (b *BoltDB) checkFlag(req FlagRequest) (val bool) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return false
+ }
+
+ key := req.Locator.URL
+ if req.UserID != "" {
+ key = req.UserID
+ }
+
+ if req.Flag == Blocked {
+ var blocked bool
+ _ = bdb.View(func(tx *bolt.Tx) error {
+ bucket := tx.Bucket([]byte(blocksBucketName))
+ v := bucket.Get([]byte(key))
+ if v == nil {
+ blocked = false
+ return nil
+ }
+
+ until, e := time.Parse(tsNano, string(v))
+ if e != nil {
+ blocked = false
+ return nil
+ }
+ blocked = time.Now().Before(until)
+ return nil
+ })
+ return blocked
+ }
+
+ _ = bdb.View(func(tx *bolt.Tx) error {
+ var bucket *bolt.Bucket
+ if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+ return err
+ }
+ val = bucket.Get([]byte(key)) != nil
+ return nil
+ })
+ return val
+}
+
+func (b *BoltDB) setFlag(req FlagRequest) (res bool, err error) {
+ bdb, e := b.db(req.Locator.SiteID)
+ if e != nil {
+ return false, e
+ }
+
+ key := req.Locator.URL
+ if req.UserID != "" {
+ key = req.UserID
+ }
+
+ err = bdb.Update(func(tx *bolt.Tx) error {
+ var bucket *bolt.Bucket
+ if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+ return err
+ }
+ switch req.Update {
+ case FlagTrue:
+ if req.Flag == Blocked {
+ val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
+ if req.TTL > 0 {
+ val = time.Now().Add(req.TTL).Format(tsNano)
+ }
+ if e = bucket.Put([]byte(key), []byte(val)); e != nil {
+ return errors.Wrapf(e, "failed to put blocked to %s", key)
+ }
+ res = true
+ return nil
+ }
+
+ if e = bucket.Put([]byte(key), []byte(time.Now().Format(tsNano))); e != nil {
+ return errors.Wrapf(e, "failed to set flag %s for %s", req.Flag, req.Locator.URL)
+ }
+ res = true
+ return nil
+ case FlagFalse:
+ if e = bucket.Delete([]byte(key)); e != nil {
+ return errors.Wrapf(e, "failed to clean flag %s for %s", req.Flag, req.Locator.URL)
+ }
+ res = false
+ }
+ return nil
+ })
+
+ return res, err
+}
+
+func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error) {
+ switch flag {
+ case ReadOnly:
+ bkt = tx.Bucket([]byte(readonlyBucketName))
+ case Blocked:
+ bkt = tx.Bucket([]byte(blocksBucketName))
+ case Verified:
+ bkt = tx.Bucket([]byte(verifiedBucketName))
+ default:
+ return nil, errors.Errorf("unsupported flag %v", flag)
+ }
+ return bkt, nil
+}
+
+func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
+
+ return bdb.Update(func(tx *bolt.Tx) error {
+
+ postBkt, e := b.getPostBucket(tx, locator.URL)
+ if e != nil {
+ return e
+ }
+
+ comment := store.Comment{}
+ if e = b.load(postBkt, commentID, &comment); e != nil {
+ return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
+ }
+ // set deleted status and clear fields
+ comment.SetDeleted(mode)
+
+ if e = b.save(postBkt, commentID, comment); e != nil {
+ return errors.Wrapf(e, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
+ }
+
+ // delete from "last" bucket
+ lastBkt := tx.Bucket([]byte(lastBucketName))
+ if e = lastBkt.Delete([]byte(commentID)); e != nil {
+ return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
+ }
+
+ // decrement comments count for post url
+ if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
+ return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
+ }
+
+ return nil
+ })
+}
+
+// deleteAll removes all top-level buckets for given siteID
+func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
+
+ // delete all buckets except blocked users
+ toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
+
+ // delete top-level buckets
+ err := bdb.Update(func(tx *bolt.Tx) error {
+ for _, bktName := range toDelete {
+
+ if e := tx.DeleteBucket([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to delete top level bucket %s", bktName)
+ }
+ if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+ }
+ }
+ return nil
+ })
+
+ return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
+}
+
+// deleteUser removes all comments for given user. Everything will be market as deleted
+// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
+func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string) error {
+ bdb, err := b.db(siteID)
+ if err != nil {
+ return err
+ }
+
+ // get list of all comments outside of transaction loop
+ posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
+ if err != nil {
+ return err
+ }
+
+ type commentInfo struct {
+ locator store.Locator
+ commentID string
+ }
+
+ // get list of commentID for all user's comment
+ comments := []commentInfo{}
+ for _, postInfo := range posts {
+ err = bdb.View(func(tx *bolt.Tx) error {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ postBkt := postsBkt.Bucket([]byte(postInfo.URL))
+ err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+ comment := store.Comment{}
+ if err = json.Unmarshal(commentVal, &comment); err != nil {
+ return errors.Wrap(err, "failed to unmarshal")
+ }
+ if comment.User.ID == userID {
+ comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
+ }
+ return nil
+ })
+ return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
+ })
+ if err != nil {
+ return err
+ }
+ }
+
+ log.Printf("[DEBUG] comments for removal=%d", len(comments))
+
+ // delete collected comments
+ for _, ci := range comments {
+ if e := b.deleteComment(bdb, ci.locator, ci.commentID, store.HardDelete); e != nil {
+ return errors.Wrapf(err, "failed to delete comment %+v", ci)
+ }
+ }
+
+ // delete user bucket
+ err = bdb.Update(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ if usersBkt != nil {
+ if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
+ return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
+ }
+ }
+ return nil
+ })
+
+ if err != nil {
+ return errors.Wrap(err, "can't delete user meta")
+ }
+
+ if len(comments) == 0 {
+ return errors.Errorf("unknown user %s", userID)
+ }
+
+ return err
+}
+
+// getPostBucket return bucket with all comments for postURL
+func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ if postsBkt == nil {
+ return nil, errors.Errorf("no bucket %s", postsBucketName)
+ }
+ res := postsBkt.Bucket([]byte(postURL))
+ if res == nil {
+ return nil, errors.Errorf("no bucket %s in store", postURL)
+ }
+ return res, nil
+}
+
+// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
+func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ if postsBkt == nil {
+ return nil, errors.Errorf("no bucket %s", postsBucketName)
+ }
+ res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
+ if err != nil {
+ return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
+ }
+ return res, nil
+}
+
+func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
+ if e != nil {
+ return nil, errors.Wrapf(e, "can't get bucket %s", userID)
+ }
+ return userIDBkt, nil
+}
+
+// save marshaled value to key for bucket. Should run in update tx
+func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
+ if value == nil {
+ return errors.Errorf("can't save nil value for %s", key)
+ }
+ jdata, jerr := json.Marshal(value)
+ if jerr != nil {
+ return errors.Wrap(jerr, "can't marshal comment")
+ }
+ if err = bkt.Put([]byte(key), jdata); err != nil {
+ return errors.Wrapf(err, "failed to save key %s", key)
+ }
+ return nil
+}
+
+// load and unmarshal json value by key from bucket. Should run in view tx
+func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
+ value := bkt.Get([]byte(key))
+ if value == nil {
+ return errors.Errorf("no value for %s", key)
+ }
+
+ if err := json.Unmarshal(value, &res); err != nil {
+ return errors.Wrap(err, "failed to unmarshal")
+ }
+ return nil
+}
+
+// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
+// it uses separate counts bucket because boltdb Stat call is very slow
+func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
+
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+
+ info := store.PostInfo{}
+ if err := b.load(infoBkt, postURL, &info); err != nil {
+ info = store.PostInfo{}
+ }
+ if val == 0 { // get current count, don't update
+ return info.Count, nil
+ }
+ info.Count += val
+
+ return info.Count, b.save(infoBkt, postURL, &info)
+}
+
+func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ info := store.PostInfo{}
+ if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
+ info = store.PostInfo{
+ Count: 0,
+ URL: comment.Locator.URL,
+ FirstTS: comment.Timestamp,
+ LastTS: comment.Timestamp,
+ }
+ }
+ info.Count++
+ info.LastTS = comment.Timestamp
+ return info, b.save(infoBkt, comment.Locator.URL, &info)
+}
+
+func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
+ if res, ok := b.dbs[siteID]; ok {
+ return res, nil
+ }
+ return nil, errors.Errorf("site %q not found", siteID)
+}
+
+// makeRef creates reference combining url and comment id
+func (b *BoltDB) makeRef(comment store.Comment) []byte {
+ return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
+}
+
+// parseRef gets parts of reference
+func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
+ elems := strings.Split(string(val), "!!")
+ if len(elems) != 2 {
+ return "", "", errors.Errorf("invalid reference value %s", string(val))
+ }
+ return elems[0], elems[1], nil
+}
diff --git a/backend/app/store/engine2/bolt_test.go b/backend/app/store/engine2/bolt_test.go
new file mode 100644
index 00000000..02db8110
--- /dev/null
+++ b/backend/app/store/engine2/bolt_test.go
@@ -0,0 +1,774 @@
+package engine2
+
+import (
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ bolt "github.com/coreos/bbolt"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestBoltDB_CreateAndFind(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, `some text, link`, res[0].Text)
+ assert.Equal(t, "user1", res[0].User.ID)
+ t.Log(res[0].ID)
+
+ _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+ assert.NotNil(t, err)
+ assert.Equal(t, "key id-1 already in store", err.Error())
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+ assert.NoError(t, b.Close())
+}
+
+func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ comment := store.Comment{
+ ID: "id-ro",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+
+ flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue}
+ v, err := b.Flag(flagReq)
+ require.NoError(t, err)
+ assert.Equal(t, true, v)
+
+ _, err = b.Create(comment)
+ assert.NotNil(t, err)
+ assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+ flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
+ v, err = b.Flag(flagReq)
+ require.NoError(t, err)
+ assert.Equal(t, false, v)
+
+ _, err = b.Create(comment)
+ assert.NoError(t, err)
+}
+
+func TestBoltDB_Get(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "2 records initially")
+
+ comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
+ assert.NoError(t, err)
+ assert.Equal(t, "some text2", comment.Text)
+
+ comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
+ assert.NotNil(t, err)
+
+ _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_Update(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "2 records initially")
+
+ comment := res[0]
+ comment.Text = "abc 123"
+ comment.Score = 100
+ err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, comment)
+ assert.NoError(t, err)
+
+ comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
+ assert.NoError(t, err)
+ assert.Equal(t, "abc 123", comment.Text)
+ assert.Equal(t, res[0].ID, comment.ID)
+ assert.Equal(t, 100, comment.Score)
+
+ err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ err = b.Update(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
+ assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
+}
+
+func TestBoltDB_FindLast(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Limit = 1
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Locator.SiteID = "bad"
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_FindLastSince(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindForUser(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, `some text, link`, res[0].Text, "second comment")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `no comments for user userZ in store`)
+}
+
+func TestBoltDB_FindForUserPagination(t *testing.T) {
+ _ = os.Remove(testDb)
+ b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+ require.Nil(t, err)
+
+ defer func() {
+ require.NoError(t, b.Close())
+ _ = os.Remove(testDb)
+ }()
+
+ c := store.Comment{
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+
+ // write 200 comments
+ for i := 0; i < 200; i++ {
+ c.ID = fmt.Sprintf("id-%d", i)
+ c.Text = fmt.Sprintf("text #%d", i)
+ c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+ _, err = b.Create(c)
+ require.Nil(t, err)
+ }
+
+ // get all comments
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 200, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+
+ // seek 0, 5 comments
+ req.Limit = 5
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+ assert.Equal(t, "id-195", res[4].ID)
+
+ // seek 10, 3 comments
+ req.Skip, req.Limit = 10, 3
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 3, len(res))
+ assert.Equal(t, "id-189", res[0].ID)
+ assert.Equal(t, "id-187", res[2].ID)
+
+ // seek 195, ask 10 comments
+ req.Skip, req.Limit = 195, 10
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-4", res[0].ID)
+ assert.Equal(t, "id-0", res[4].ID)
+
+ // seek 255, ask 10 comments
+ req.Skip, req.Limit = 255, 10
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_CountPost(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+ c, err := b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+ c, err = b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c)
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_CountUser(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+ c, err := b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`)
+}
+
+func TestBoltDB_InfoPost(t *testing.T) {
+ b, teardown := prep(t) // two comments for https://radio-t.com
+ defer teardown()
+
+ ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
+
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err := b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+ ReadOnly: true}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ _, err = b.Info(req)
+ require.NotNil(t, err)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+ _, err = b.Info(req)
+ require.NotNil(t, err)
+
+ fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
+ _, err = b.Flag(fr)
+ require.NoError(t, err)
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+ ReadOnly: true}}, r)
+}
+
+func TestBoltDB_InfoList(t *testing.T) {
+ b, teardown := prep(t) // two comments for https://radio-t.com
+ defer teardown()
+
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.Nil(t, err)
+
+ ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
+
+ req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+ res, err := b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+ res, err = b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+ res, err = b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+ res, err = b.Info(req)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+ _, err = b.Info(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_FlagBlockedUser(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+ val, err := b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing blocked yet")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.True(t, val, "user1 blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.True(t, val, "user1 still blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.False(t, val, "user1 unblocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err, "non-existing user can't be blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing blocked on wrong site")
+}
+
+func TestBolt_FlagReadOnlyPost(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err := b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing ro")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.True(t, val, "url-1 ro")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "url-2 still writable")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "url-1 writable")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing ro on wrong site")
+}
+
+func TestBolt_FlagVerified(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ isVerified := func(site, user string) bool {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+ v, err := b.Flag(req)
+ require.NoError(t, err)
+ return v
+ }
+
+ setVerified := func(site, user string, status FlagStatus) error {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+ assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+ assert.NoError(t, setVerified("radio-t", "u1", FlagFalse))
+ assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+ assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`)
+ assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse))
+
+ assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u3", FlagFalse))
+}
+
+func TestBolt_FlagListVerified(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ toIDs := func(inp []interface{}) (res []string) {
+ res = make([]string, len(inp))
+ for i, v := range inp {
+ vv, ok := v.(string)
+ require.True(t, ok)
+ res[i] = vv
+ }
+ return res
+ }
+
+ setVerified := func(site, user string, status FlagStatus) error {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ ids, err := b.ListFlags("radio-t", Verified)
+ assert.NoError(t, err)
+ assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+ ids, err = b.ListFlags("radio-t", Verified)
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+ _, err = b.ListFlags("radio-t-bad", Verified)
+ assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
+}
+
+func TestBolt_FlagListBlocked(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error {
+ req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+ res = make([]store.BlockedUser, len(inp))
+ for i, v := range inp {
+ vv, ok := v.(store.BlockedUser)
+ require.True(t, ok)
+ res[i] = vv
+ }
+ return res
+ }
+ assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
+ assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond))
+ assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
+
+ vv, err := b.ListFlags("radio-t", Blocked)
+ assert.NoError(t, err)
+
+ blockedList := toBlocked(vv)
+ assert.Equal(t, 2, len(blockedList))
+ assert.Equal(t, "user1", blockedList[0].ID)
+ assert.Equal(t, "user2", blockedList[1].ID)
+ t.Logf("%+v", blockedList)
+
+ // check block expiration
+ time.Sleep(50 * time.Millisecond)
+ vv, err = b.ListFlags("radio-t", Blocked)
+ assert.NoError(t, err)
+ blockedList = toBlocked(vv)
+ assert.Equal(t, 1, len(blockedList))
+ assert.Equal(t, "user1", blockedList[0].ID)
+
+ _, err = b.ListFlags("bad", Blocked)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+}
+
+func TestBolt_DeleteComment(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ count, err := b.Count(reqReq)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count, "count=2 initially")
+
+ delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+ err = b.Delete(delReq)
+ assert.NoError(t, err)
+
+ res, err = b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+ assert.Equal(t, "some text2", res[1].Text)
+ assert.False(t, res[1].Deleted)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+ count, err = b.Count(reqReq)
+ require.NoError(t, err)
+ assert.Equal(t, 1, count)
+
+ delReq.CommentID = "123456"
+ err = b.Delete(delReq)
+ assert.NotNil(t, err)
+
+ delReq.Locator.SiteID = "bad"
+ delReq.CommentID = res[0].ID
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
+}
+
+func TestBolt_DeleteHard(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ CommentID: res[0].ID, DeleteMode: store.HardDelete}
+ err = b.Delete(delReq)
+ assert.NoError(t, err)
+
+ res, err = b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestBolt_DeleteAll(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+ err := b.Delete(delReq)
+ assert.NoError(t, err)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}}
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUser(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ require.NoError(t, err)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+ c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c, "0 count")
+
+ _, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+ assert.EqualError(t, err, "no comments for user user1 in store")
+
+ comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+ assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltDB_ref(t *testing.T) {
+ b := BoltDB{}
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ res := b.makeRef(comment)
+ assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
+
+ url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
+ assert.NoError(t, err)
+ assert.Equal(t, "https://radio-t.com/2", url)
+ assert.Equal(t, "12345", id)
+
+ _, _, err = b.parseRef([]byte("https://radio-t.com/2"))
+ assert.NotNil(t, err)
+}
+
+func TestBoltDB_NewFailed(t *testing.T) {
+ _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
+ assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
+ _ = os.Remove(testDb)
+
+ boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+ assert.Nil(t, err)
+ b = boltStore
+
+ comment := store.Comment{
+ ID: "id-1",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ comment = store.Comment{
+ ID: "id-2",
+ Text: "some text2",
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ teardown = func() {
+ require.NoError(t, b.Close())
+ _ = os.Remove(testDb)
+ }
+ return b, teardown
+}
diff --git a/backend/app/store/engine2/engine.go b/backend/app/store/engine2/engine.go
new file mode 100644
index 00000000..eb3a6e8c
--- /dev/null
+++ b/backend/app/store/engine2/engine.go
@@ -0,0 +1,128 @@
+package engine2
+
+// Package engine defines interfaces each supported storage should implement.
+// Includes default implementation with boltdb
+
+import (
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
+//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
+
+// Interface defines methods provided by low-level storage engine
+type Interface interface {
+ Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
+ Update(locator store.Locator, comment store.Comment) error // update comment, mutable parts only
+ Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id
+ Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site
+ Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info
+ Count(req FindRequest) (int, error) // get count for post or user
+ Delete(req DeleteRequest) error // delete post(s) by id or by userID
+ Flag(req FlagRequest) (bool, error) // set and get flags
+ ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user
+ Close() error // close storage engine
+}
+
+// FindRequest is the input for all find operations
+type FindRequest struct {
+ Locator store.Locator // lack of URL means site operation
+ UserID string // presence of UserID treated as user-related find
+ Sort string // sort order with +/-field syntax
+ Since time.Time // time limit for found results
+ Limit, Skip int
+}
+
+// InfoRequest is the input of Info operation used to get meta data about posts
+type InfoRequest struct {
+ Locator store.Locator
+ Limit, Skip int
+ ReadOnlyAge int
+}
+
+type DeleteRequest struct {
+ Locator store.Locator // lack of URL means site operation
+ CommentID string
+ UserID string
+ DeleteMode store.DeleteMode
+}
+
+// Flag defines type of binary attribute
+type Flag string
+
+// FlagStatus represents values of the flag update
+type FlagStatus int
+
+// enum of update values
+const (
+ FlagNonSet FlagStatus = 0
+ FlagTrue FlagStatus = 1
+ FlagFalse FlagStatus = -1
+)
+
+// Enum of all flags
+const (
+ ReadOnly = Flag("readonly")
+ Verified = Flag("verified")
+ Blocked = Flag("blocked")
+)
+
+// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
+type FlagRequest struct {
+ Flag Flag // flag type
+ Locator store.Locator // post locator
+ UserID string // for flags setting user status
+ Update FlagStatus // if FlagNonSet it will be get op, if set will set the value
+ TTL time.Duration // ttl for time-sensitive flags only, like blocked for some period
+}
+
+const (
+ // limits
+ lastLimit = 1000
+ userLimit = 500
+)
+
+// SortComments is for engines can't sort data internally
+func SortComments(comments []store.Comment, sortFld string) []store.Comment {
+ sort.Slice(comments, func(i, j int) bool {
+ switch sortFld {
+ case "+time", "-time", "time", "+active", "-active", "active":
+ if strings.HasPrefix(sortFld, "-") {
+ return comments[i].Timestamp.After(comments[j].Timestamp)
+ }
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+
+ case "+score", "-score", "score":
+ if strings.HasPrefix(sortFld, "-") {
+ if comments[i].Score == comments[j].Score {
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+ }
+ return comments[i].Score > comments[j].Score
+ }
+ if comments[i].Score == comments[j].Score {
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+ }
+ return comments[i].Score < comments[j].Score
+
+ case "+controversy", "-controversy", "controversy":
+ if strings.HasPrefix(sortFld, "-") {
+ if comments[i].Controversy == comments[j].Controversy {
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+ }
+ return comments[i].Controversy > comments[j].Controversy
+ }
+ if comments[i].Controversy == comments[j].Controversy {
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+ }
+ return comments[i].Controversy < comments[j].Controversy
+
+ default:
+ return comments[i].Timestamp.Before(comments[j].Timestamp)
+ }
+ })
+ return comments
+}
diff --git a/backend/app/store/engine2/engine_test.go b/backend/app/store/engine2/engine_test.go
new file mode 100644
index 00000000..a17585b9
--- /dev/null
+++ b/backend/app/store/engine2/engine_test.go
@@ -0,0 +1,55 @@
+package engine2
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+func TestEngine_sortComments(t *testing.T) {
+ cc := []store.Comment{
+ {ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
+ {ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
+ {ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
+ {ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
+ }
+
+ SortComments(cc, "+time")
+ assert.Equal(t, "1", cc[0].ID)
+ assert.Equal(t, "2", cc[1].ID)
+ assert.Equal(t, "3", cc[2].ID)
+ assert.Equal(t, "4", cc[3].ID)
+
+ SortComments(cc, "-time")
+ assert.Equal(t, "4", cc[0].ID)
+ assert.Equal(t, "3", cc[1].ID)
+ assert.Equal(t, "2", cc[2].ID)
+ assert.Equal(t, "1", cc[3].ID)
+
+ SortComments(cc, "score")
+ assert.Equal(t, "2", cc[0].ID)
+ assert.Equal(t, "1", cc[1].ID)
+ assert.Equal(t, "3", cc[2].ID)
+ assert.Equal(t, "4", cc[3].ID)
+
+ SortComments(cc, "-score")
+ assert.Equal(t, "3", cc[0].ID)
+ assert.Equal(t, "4", cc[1].ID)
+ assert.Equal(t, "1", cc[2].ID)
+ assert.Equal(t, "2", cc[3].ID)
+
+ SortComments(cc, "controversy")
+ assert.Equal(t, "1", cc[0].ID)
+ assert.Equal(t, "4", cc[1].ID)
+ assert.Equal(t, "2", cc[2].ID)
+ assert.Equal(t, "3", cc[3].ID)
+
+ SortComments(cc, "-controversy")
+ assert.Equal(t, "3", cc[0].ID)
+ assert.Equal(t, "2", cc[1].ID)
+ assert.Equal(t, "1", cc[2].ID)
+ assert.Equal(t, "4", cc[3].ID)
+}
diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go
new file mode 100644
index 00000000..ff805465
--- /dev/null
+++ b/backend/app/store/remote/remote.go
@@ -0,0 +1,134 @@
+package remote
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// Client implements remote engine and delegates all calls to remote http server
+type Client struct {
+ API string
+ Client http.Client
+ AuthUser string
+ AuthPasswd string
+}
+
+// Request encloses method name and all params
+type Request struct {
+ Method string `json:"method"`
+ Params interface{} `json:"params"`
+}
+
+// Response encloses result and error received from remote server
+type Response struct {
+ Result *json.RawMessage `json:"result,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+// Create comment and return ID
+func (r *Client) Create(comment store.Comment) (commentID string, err error) {
+
+ resp, err := r.call("create", comment)
+ if err != nil {
+ return "", err
+ }
+
+ err = json.Unmarshal(*resp.Result, &commentID)
+ return commentID, err
+}
+
+// Get comment by ID
+func (r *Client) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
+ resp, err := r.call("get", locator, commentID)
+ if err != nil {
+ return store.Comment{}, err
+ }
+
+ err = json.Unmarshal(*resp.Result, &comment)
+ return comment, err
+}
+
+// Put updates comment, mutable parts only
+func (r *Client) Put(locator store.Locator, comment store.Comment) error {
+ _, err := r.call("put", locator, comment)
+ return err
+}
+
+// Find comments for locator
+func (r *Client) Find(locator store.Locator, sort string) (comments []store.Comment, err error) {
+ resp, err := r.call("find", locator, sort)
+ if err != nil {
+ return []store.Comment{}, err
+ }
+ err = json.Unmarshal(*resp.Result, &comments)
+ return comments, err
+}
+
+// Last comments for given site, sorted by time
+func (r *Client) Last(siteID string, limit int, since time.Time) (comments []store.Comment, err error) {
+ resp, err := r.call("last", siteID, limit, since)
+ if err != nil {
+ return []store.Comment{}, err
+ }
+ err = json.Unmarshal(*resp.Result, &comments)
+ return comments, err
+}
+
+// User get comments by user, sorted by time
+func (r *Client) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+ resp, err := r.call("user", siteID, userID, limit, skip)
+ if err != nil {
+ return []store.Comment{}, err
+ }
+ err = json.Unmarshal(*resp.Result, &comments)
+ return comments, err
+}
+
+// UserCount gets comments count by user
+func (r *Client) UserCount(siteID, userID string) (count int, err error) {
+ resp, err := r.call("user_count", siteID, userID)
+ if err != nil {
+ return 0, err
+ }
+ err = json.Unmarshal(*resp.Result, &count)
+ return count, err
+}
+
+func (r *Client) call(method string, args ...interface{}) (*Response, error) {
+
+ b, err := json.Marshal(Request{Method: method, Params: args})
+ if err != nil {
+ return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+ }
+
+ req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b))
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to make request for %s", method)
+ }
+
+ req.SetBasicAuth(r.AuthUser, r.AuthPasswd)
+ resp, err := r.Client.Do(req)
+ if err != nil {
+ return nil, errors.Wrapf(err, "remote call failed for %s", method)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, errors.Errorf("bad status %d for %s", resp.StatusCode, method)
+ }
+
+ cr := Response{}
+ if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil {
+ return nil, errors.Wrapf(err, "failed to decode response for %s", method)
+ }
+
+ if cr.Error != "" {
+ return nil, errors.New(cr.Error)
+ }
+ return &cr, nil
+}
diff --git a/backend/app/store/remote/remote_test.go b/backend/app/store/remote/remote_test.go
new file mode 100644
index 00000000..d3e56f9d
--- /dev/null
+++ b/backend/app/store/remote/remote_test.go
@@ -0,0 +1,133 @@
+package remote
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+func TestClient_Create(t *testing.T) {
+ ts := testServer(t, `{"method":"create","params":[{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{"result":"12345"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"},
+ Text: "msg"})
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestClient_Get(t *testing.T) {
+ ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`,
+ `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ res, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site")
+ assert.NoError(t, err)
+ assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestClient_GetWithErrorResult(t *testing.T) {
+ ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, `{"error":"failed"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site")
+ assert.EqualError(t, err, "failed")
+}
+
+func TestClient_GetWithErrorDecode(t *testing.T) {
+ ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, ``)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site")
+ assert.EqualError(t, err, "failed to decode response for get: EOF")
+}
+
+func TestClient_GetWithErrorRemote(t *testing.T) {
+ c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}
+
+ _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site")
+ assert.NotNil(t, err)
+ assert.True(t, strings.Contains(err.Error(), "remote call failed for get:"))
+}
+
+func TestClient_FailedStatus(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ t.Logf("req: %s", string(body))
+ w.WriteHeader(400)
+ }))
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site")
+ assert.EqualError(t, err, "bad status 400 for get")
+}
+
+func TestClient_Put(t *testing.T) {
+ ts := testServer(t, `{"method":"put","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ err := c.Put(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123",
+ Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"})
+ assert.NoError(t, err)
+
+}
+
+func TestClient_Find(t *testing.T) {
+ ts := testServer(t, `{"method":"find","params":[{"url":"http://example.com/url"},""]}`,
+ `{"result":[{"text":"1"},{"text":"2"}]}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ res, err := c.Find(store.Locator{URL: "http://example.com/url"}, "")
+ assert.NoError(t, err)
+ assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func TestClient_Last(t *testing.T) {
+ ts := testServer(t, `{"method":"last","params":["site1",100,"2019-06-06T19:34:10Z"]}`,
+ `{"result":[{"text":"1"},{"text":"2"}]}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ res, err := c.Last("site1", 100, time.Date(2019, 6, 6, 19, 34, 10, 0, time.UTC))
+ assert.NoError(t, err)
+ assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func TestClient_User(t *testing.T) {
+ ts := testServer(t, `{"method":"user","params":["site1","u1",100,4]}`, `{"result":[{"text":"1"},{"text":"2"}]}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+
+ res, err := c.User("site1", "u1", 100, 4)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.Equal(t, req, string(body))
+ t.Logf("req: %s", string(body))
+ fmt.Fprintf(w, resp)
+ }))
+}
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 6719e042..9bb5ab42 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -19,12 +19,13 @@ import (
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
+ "github.com/umputun/remark/backend/app/store/engine2"
"github.com/umputun/remark/backend/app/store/image"
)
// DataStore wraps store.Interface with additional methods
type DataStore struct {
- engine.Interface
+ Engine engine2.Interface
EditDuration time.Duration
AdminStore admin.Store
MaxCommentSize int
@@ -98,12 +99,13 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
}()
s.submitImages(comment)
- return s.Interface.Create(comment)
+ return s.Engine.Create(comment)
}
// Find wraps engine's Find call and alter results if needed
func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.Find(locator, sort)
+ req := engine2.FindRequest{Locator: locator, Sort: sort}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
@@ -130,19 +132,24 @@ func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([
// Get comment by ID
func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) {
- c, err := s.Interface.Get(locator, commentID)
+ c, err := s.Engine.Get(locator, commentID)
if err != nil {
return store.Comment{}, err
}
return s.alterComment(c, user), nil
}
+// Put updates comment, mutable parts only
+func (s *DataStore) Put(locator store.Locator, comment store.Comment) error {
+ return s.Engine.Update(locator, comment)
+}
+
// submitImages initiated delayed commit of all images from the comment uploaded to remark42
func (s *DataStore) submitImages(comment store.Comment) {
s.ImageService.Submit(func() []string {
c := comment
- cc, err := s.Interface.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+ cc, err := s.Engine.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
if err != nil {
log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
return nil
@@ -182,14 +189,20 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err
return comment, nil
}
+// DeleteAll removes all data from site
+func (s *DataStore) DeleteAll(siteID string) error {
+ req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}}
+ return s.Engine.Delete(req)
+}
+
// SetPin pin/un-pin comment as special
func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
- comment, err := s.Interface.Get(locator, commentID)
+ comment, err := s.Engine.Get(locator, commentID)
if err != nil {
return err
}
comment.Pin = status
- return s.Put(locator, comment)
+ return s.Engine.Update(locator, comment)
}
// Vote for comment by id and locator
@@ -199,7 +212,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
cLock.Lock() // prevents race on voting
defer cLock.Unlock()
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(locator, commentID)
if err != nil {
return comment, err
}
@@ -258,7 +271,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
comment.Controversy = s.controversy(s.upsAndDowns(comment))
- return comment, s.Put(locator, comment)
+ return comment, s.Engine.Update(locator, comment)
}
// controversy calculates controversial index of votes
@@ -287,7 +300,7 @@ type EditRequest struct {
// EditComment to edit text and update Edit info
func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(locator, commentID)
if err != nil {
return comment, err
}
@@ -303,7 +316,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
if req.Delete { // delete request
comment.Deleted = true
- return comment, s.Delete(locator, commentID, store.SoftDelete)
+ delReq := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete}
+ return comment, s.Engine.Delete(delReq)
}
if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) {
@@ -318,7 +332,7 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
}
comment.Sanitize()
- err = s.Put(locator, comment)
+ err = s.Engine.Update(locator, comment)
return comment, err
}
@@ -336,7 +350,8 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
return true
}
- comments, err := s.Interface.Last(comment.Locator.SiteID, maxLastCommentsReply, time.Time{})
+ req := engine2.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply}
+ comments, err := s.Engine.Find(req)
if err != nil {
log.Printf("[WARN] can't get last comments for reply check, %v", err)
return false
@@ -395,7 +410,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
return comment, errors.New("no title extractor")
}
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(locator, commentID)
if err != nil {
return comment, err
}
@@ -406,7 +421,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
return comment, err
}
comment.PostTitle = title
- err = s.Put(locator, comment)
+ err = s.Engine.Update(locator, comment)
return comment, err
}
@@ -414,7 +429,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
res := []store.PostInfo{}
for _, p := range postIDs {
- if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
+ req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}}
+ if c, err := s.Engine.Count(req); err == nil {
res = append(res, store.PostInfo{URL: p, Count: c})
}
}
@@ -449,20 +465,126 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
return false
}
+// IsReadOnly checks if post read-only
+func (s *DataStore) IsReadOnly(locator store.Locator) bool {
+ req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetReadOnly set/reset read-only flag
+func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error {
+ roStatus := engine2.FlagFalse
+ if status {
+ roStatus = engine2.FlagTrue
+
+ }
+ req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly, Update: roStatus}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// IsVerified checks if user verified
+func (s *DataStore) IsVerified(siteID string, userID string) bool {
+ req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetVerified set/reset verified status for user
+func (s *DataStore) SetVerified(siteID string, userID string, status bool) error {
+ roStatus := engine2.FlagFalse
+ if status {
+ roStatus = engine2.FlagTrue
+ }
+ req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified, Update: roStatus}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// IsBlocked checks if user blocked
+func (s *DataStore) IsBlocked(siteID string, userID string) bool {
+ req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Blocked}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetBlock set/reset verified status for user
+func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
+ roStatus := engine2.FlagFalse
+ if status {
+ roStatus = engine2.FlagTrue
+ }
+ req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID,
+ Flag: engine2.Blocked, Update: roStatus, TTL: ttl}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// Blocked returns list with all blocked users
+func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) {
+ blocked, e := s.Engine.ListFlags(siteID, engine2.Blocked)
+ if e != nil {
+ return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID)
+ }
+ for _, v := range blocked {
+ res = append(res, v.(store.BlockedUser))
+ }
+ return res, nil
+}
+
+// Info get post info
+func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
+ req := engine2.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge}
+ res, err := s.Engine.Info(req)
+ if err != nil {
+ return store.PostInfo{}, err
+ }
+ if len(res) == 0 {
+ return store.PostInfo{}, errors.Errorf("post %+v not found", locator)
+ }
+ return res[0], nil
+}
+
+// Delete comment by id
+func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
+ req := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
+ return s.Engine.Delete(req)
+}
+
+// DeleteUser removes all comments from user
+func (s *DataStore) DeleteUser(siteID string, userID string) error {
+ req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: store.HardDelete}
+ return s.Engine.Delete(req)
+}
+
+// List of commented posts
+func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
+ req := engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip}
+ return s.Engine.Info(req)
+}
+
+// Count gets number of comments for the post
+func (s *DataStore) Count(locator store.Locator) (int, error) {
+ req := engine2.FindRequest{Locator: locator}
+ return s.Engine.Count(req)
+}
+
// Metas returns metadata for users and posts
func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
umetas = []UserMetaData{}
pmetas = []PostMetaData{}
+
// set posts meta
- posts, err := s.List(siteID, 0, 0)
+ posts, err := s.Engine.Info(engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}})
if err != nil {
return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID)
}
+
for _, p := range posts {
if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) {
pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true})
}
-
}
// set users meta
@@ -484,11 +606,12 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
}
// process verified users
- verified, err := s.Verified(siteID)
+ verified, err := s.Engine.ListFlags(siteID, engine2.Verified)
if err != nil {
return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID)
}
- for _, v := range verified {
+ for _, vi := range verified {
+ v := vi.(string)
val, ok := m[v]
if !ok {
val = UserMetaData{ID: v}
@@ -531,22 +654,35 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
// User gets comment for given userID on siteID
func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.User(siteID, userID, limit, skip)
+ req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
return s.alterComments(comments, user), nil
}
+// UserCount is comments count by user
+func (s *DataStore) UserCount(siteID, userID string) (int, error) {
+ req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID}
+ return s.Engine.Count(req)
+}
+
// Last gets last comments for site, cross-post. Limited by count and optional since ts
func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.Last(siteID, limit, since)
+ req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
return s.alterComments(comments, user), nil
}
+// Close store service
+func (s *DataStore) Close() error {
+ return s.Engine.Close()
+}
+
func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
for _, v := range c.Votes {
if v {
@@ -583,7 +719,9 @@ func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []st
func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) {
- blocked := s.IsBlocked(c.Locator.SiteID, c.User.ID)
+ blocReq := engine2.FlagRequest{Flag: engine2.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+ blocked, _ := s.Engine.Flag(blocReq)
+
// process blocked users
if blocked {
if !user.Admin { // reset comment to deleted for non-admins
@@ -595,7 +733,8 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co
// set verified status retroactively
if !blocked {
- c.User.Verified = s.IsVerified(c.Locator.SiteID, c.User.ID)
+ verifReq := engine2.FlagRequest{Flag: engine2.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+ c.User.Verified, _ = s.Engine.Flag(verifReq)
}
// hide info from non-admins