mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clear mongo leftovers

Browse Source
This commit is contained in:
Umputun
2019-06-25 20:06:30 -05:00
parent 82ce19fd3c
commit 01bdf4d839
18 changed files with 11 additions and 3104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+2 -1
View File
@@ -20,4 +20,5 @@ debug.test
*.prof
*.test
remark42
/backend/var/
/backend/var/
compose-private-backend.yml
.gitignore
+1
View File
@@ -19,3 +19,4 @@ debug.test
remark42
/bin/
/backend/var/
compose-private-backend.yml
.travis.yml
-6
View File
@@ -3,9 +3,6 @@ install:
- docker-compose --version
script:
- docker run -d --name=mongo mongo:3.6 && sleep 3
- export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
- echo "running mongo on $MONGO_TEST"
- docker build
--build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
--build-arg CI=$CI
@@ -19,7 +16,4 @@ script:
--build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
--build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
--build-arg TRAVIS_TAG=$TRAVIS_TAG
--build-arg MONGO_TEST=$MONGO_TEST
.
- docker rm -f mongo
Dockerfile
+1 -6
View File
@@ -19,7 +19,6 @@ ARG DRONE_BRANCH
ARG DRONE_PULL_REQUEST
ARG SKIP_BACKEND_TEST
ARG MONGO_TEST
ADD backend /build/backend
ADD .git /build/.git
@@ -29,18 +28,14 @@ ENV GOFLAGS="-mod=vendor"
# run tests
RUN \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then \
go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \
go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
else echo "skip backend test" ; fi
RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
# linters
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
--enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
--enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
backend/app/cmd/avatar.go
-1
View File
@@ -16,7 +16,6 @@ import (
type AvatarCommand struct {
AvatarSrc AvatarGroup `group:"src" namespace:"src"`
AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
// Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
migrator AvatarMigrator
CommonOpts
backend/app/cmd/avatar_test.go
+1 -8
View File
@@ -13,13 +13,6 @@ import (
func TestAvatar_Execute(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
defer os.RemoveAll("/tmp/ava-test")
// from fs to bolt
@@ -37,7 +30,7 @@ func TestAvatar_Execute(t *testing.T) {
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
"--dst.bolt.file=/tmp/ava-test.db"})
"--dst.bolt.file=/tmp/ava-test2.db"})
require.NoError(t, err)
err = cmd.Execute(nil)
assert.Error(t, err, "failed blah")
backend/app/cmd/server.go
+4 -4
View File
@@ -98,7 +98,7 @@ type StoreGroup struct {
// ImageGroup defines options group for store pictures
type ImageGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -114,7 +114,7 @@ type ImageGroup struct {
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
@@ -126,7 +126,7 @@ type AvatarGroup struct {
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -176,7 +176,7 @@ type StreamGroup struct {
// RemoteGroup defines options for remote modules (plugins)
type RemoteGroup struct {
API string `long:"api" env:"API" description:"remote extension api url"`
TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"`
TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
AuthUser string `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
}
backend/app/store/engine_old/bolt_accessor.go
-550
View File
@@ -1,550 +0,0 @@
package engine_old
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"time"
bolt "github.com/coreos/bbolt"
log "github.com/go-pkgz/lgr"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
// there are 5 types of top-level buckets:
// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
// value is not full comment but a reference combined from post-url+commentID
// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
// is a nested bucket named userID with kv as ts:reference
// - blocking info sits in "block" bucket. Key is userID, value - ts
// - counts per post to keep number of comments. Key is post url, value - count
// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
type BoltDB struct {
dbs map[string]*bolt.DB
}
const (
// top level buckets
postsBucketName = "posts"
lastBucketName = "last"
userBucketName = "users"
blocksBucketName = "block"
infoBucketName = "info"
readonlyBucketName = "readonly"
verifiedBucketName = "verified"
tsNano = "2006-01-02T15:04:05.000000000Z07:00"
)
// BoltSite defines single site param
type BoltSite struct {
FileName string // full path to boltdb
SiteID string // ID to access given site
}
// NewBoltDB makes persistent boltdb-based store
func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
result := BoltDB{dbs: make(map[string]*bolt.DB)}
for _, site := range sites {
db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
}
// make top-level buckets
topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
infoBucketName, readonlyBucketName, verifiedBucketName}
err = db.Update(func(tx *bolt.Tx) error {
for _, bktName := range topBuckets {
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
}
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "failed to create top level bucket)")
}
result.dbs[site.SiteID] = db
log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
}
return &result, nil
}
// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
bdb, err := b.db(comment.Locator.SiteID)
if err != nil {
return "", err
}
if b.IsReadOnly(comment.Locator) {
return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
}
err = bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
if e != nil {
return e
}
// check if key already in store, reject doubles
if postBkt.Get([]byte(comment.ID)) != nil {
return errors.Errorf("key %s already in store", comment.ID)
}
// serialize comment to json []byte for bolt and save
if e = b.save(postBkt, comment.ID, comment); e != nil {
return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
}
ref := b.makeRef(comment)
// add reference to comment to "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
commentTs := []byte(comment.Timestamp.Format(tsNano))
e = lastBkt.Put(commentTs, ref)
if e != nil {
return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
}
// add reference to commentID to "users" bucket
userBkt, e := b.getUserBucket(tx, comment.User.ID)
if e != nil {
return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
}
// put into individual user's bucket with ts as a key
if e = userBkt.Put(commentTs, ref); e != nil {
return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
}
// set info with the count for post url
if _, e = b.setInfo(tx, comment); e != nil {
return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
}
return nil
})
return comment.ID, err
}
// Find returns all comments for post and sorts results
func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
comments = []store.Comment{}
bdb, err := b.db(locator.SiteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
return bucket.ForEach(func(k, v []byte) error {
comment := store.Comment{}
if e = json.Unmarshal(v, &comment); e != nil {
return errors.Wrap(e, "failed to unmarshal")
}
comments = append(comments, comment)
return nil
})
})
comments = SortComments(comments, sortFld)
return comments, err
}
// Last returns up to max last comments for given siteID
func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
comments = []store.Comment{}
if max > lastLimit || max == 0 {
max = lastLimit
}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
lastBkt := tx.Bucket([]byte(lastBucketName))
c := lastBkt.Cursor()
for k, v := c.Last(); k != nil; k, v = c.Prev() {
if !since.IsZero() {
// stop if reached "since" ts
tsSince := []byte(since.Format(tsNano))
if bytes.Compare(k, tsSince) <= 0 {
break
}
}
url, commentID, e := b.parseRef(v)
if e != nil {
return e
}
postBkt, e := b.getPostBucket(tx, url)
if e != nil {
return e
}
comment := store.Comment{}
if e = b.load(postBkt, commentID, &comment); e != nil {
log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
continue
}
if comment.Deleted {
continue
}
comments = append(comments, comment)
if len(comments) >= max {
break
}
}
return nil
})
return comments, err
}
// Count returns number of comments for locator
func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
bdb, err := b.db(locator.SiteID)
if err != nil {
return 0, err
}
err = bdb.View(func(tx *bolt.Tx) error {
var e error
count, e = b.count(tx, locator.URL, 0)
return e
})
return count, err
}
// List returns list of all commented posts with counters
// uses count bucket to get number of comments
func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
postsBkt := tx.Bucket([]byte(postsBucketName))
c := postsBkt.Cursor()
n := 0
for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
n++
if skip > 0 && n <= skip {
continue
}
postURL := string(k)
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
if e := b.load(infoBkt, postURL, &info); e != nil {
return errors.Wrapf(e, "can't load info for %s", postURL)
}
list = append(list, info)
if limit > 0 && len(list) >= limit {
break
}
}
return nil
})
return list, err
}
// Info returns time range and count for locator
func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
bdb, err := b.db(locator.SiteID)
if err != nil {
return store.PostInfo{}, err
}
info := store.PostInfo{}
err = bdb.View(func(tx *bolt.Tx) error {
infoBkt := tx.Bucket([]byte(infoBucketName))
if e := b.load(infoBkt, locator.URL, &info); e != nil {
return errors.Wrapf(e, "can't load info for %s", locator.URL)
}
return nil
})
// set read-only from age and manual bucket
info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
if b.IsReadOnly(locator) {
info.ReadOnly = true
}
return info, err
}
// User extracts all comments for given site and given userID
// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
comments = []store.Comment{}
commentRefs := []string{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
if limit == 0 || limit > userLimit {
limit = userLimit
}
// get list of references to comments
err = bdb.View(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
userIDBkt := usersBkt.Bucket([]byte(userID))
if userIDBkt == nil {
return errors.Errorf("no comments for user %s in store", userID)
}
c := userIDBkt.Cursor()
skipComments := 0
for k, v := c.Last(); k != nil; k, v = c.Prev() {
if len(commentRefs) >= limit {
break
}
if skip > 0 && skipComments < skip {
skipComments++
continue
}
commentRefs = append(commentRefs, string(v))
}
return nil
})
if err != nil {
return comments, err
}
// retrieve comments for refs
for _, v := range commentRefs {
url, commentID, errParse := b.parseRef([]byte(v))
if errParse != nil {
return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
}
if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
comments = append(comments, c)
}
}
return comments, err
}
// UserCount returns number of comments for user
func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
bdb, err := b.db(siteID)
if err != nil {
return 0, err
}
count := 0
err = bdb.View(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
userIDBkt := usersBkt.Bucket([]byte(userID))
if userIDBkt == nil {
return errors.Errorf("no comments for user %s in store", userID)
}
stats := userIDBkt.Stats()
count = stats.KeyN
return nil
})
return count, err
}
// Get returns comment for locator.URL and commentID string
func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
bdb, err := b.db(locator.SiteID)
if err != nil {
return comment, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
return b.load(bucket, commentID, &comment)
})
return comment, err
}
// Put updates comment for locator.URL with mutable part of comment
func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
if curComment, err := b.Get(locator, comment.ID); err == nil {
// preserve immutable fields
comment.ParentID = curComment.ParentID
comment.Locator = curComment.Locator
comment.Timestamp = curComment.Timestamp
comment.User = curComment.User
}
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
return b.save(bucket, comment.ID, comment)
})
}
// Close boltdb store
func (b *BoltDB) Close() error {
errs := new(multierror.Error)
for site, db := range b.dbs {
err := errors.Wrapf(db.Close(), "can't close site %s", site)
errs = multierror.Append(errs, err)
}
return errs.ErrorOrNil()
}
// getPostBucket return bucket with all comments for postURL
func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
postsBkt := tx.Bucket([]byte(postsBucketName))
if postsBkt == nil {
return nil, errors.Errorf("no bucket %s", postsBucketName)
}
res := postsBkt.Bucket([]byte(postURL))
if res == nil {
return nil, errors.Errorf("no bucket %s in store", postURL)
}
return res, nil
}
// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
postsBkt := tx.Bucket([]byte(postsBucketName))
if postsBkt == nil {
return nil, errors.Errorf("no bucket %s", postsBucketName)
}
res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
if err != nil {
return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
}
return res, nil
}
func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
usersBkt := tx.Bucket([]byte(userBucketName))
userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
if e != nil {
return nil, errors.Wrapf(e, "can't get bucket %s", userID)
}
return userIDBkt, nil
}
// save marshaled value to key for bucket. Should run in update tx
func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
if value == nil {
return errors.Errorf("can't save nil value for %s", key)
}
jdata, jerr := json.Marshal(value)
if jerr != nil {
return errors.Wrap(jerr, "can't marshal comment")
}
if err = bkt.Put([]byte(key), jdata); err != nil {
return errors.Wrapf(err, "failed to save key %s", key)
}
return nil
}
// load and unmarshal json value by key from bucket. Should run in view tx
func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
value := bkt.Get([]byte(key))
if value == nil {
return errors.Errorf("no value for %s", key)
}
if err := json.Unmarshal(value, &res); err != nil {
return errors.Wrap(err, "failed to unmarshal")
}
return nil
}
// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
// it uses separate counts bucket because boltdb Stat call is very slow
func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
if err := b.load(infoBkt, postURL, &info); err != nil {
info = store.PostInfo{}
}
if val == 0 { // get current count, don't update
return info.Count, nil
}
info.Count += val
return info.Count, b.save(infoBkt, postURL, &info)
}
func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
infoBkt := tx.Bucket([]byte(infoBucketName))
info := store.PostInfo{}
if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
info = store.PostInfo{
Count: 0,
URL: comment.Locator.URL,
FirstTS: comment.Timestamp,
LastTS: comment.Timestamp,
}
}
info.Count++
info.LastTS = comment.Timestamp
return info, b.save(infoBkt, comment.Locator.URL, &info)
}
func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
if res, ok := b.dbs[siteID]; ok {
return res, nil
}
return nil, errors.Errorf("site %q not found", siteID)
}
// makeRef creates reference combining url and comment id
func (b *BoltDB) makeRef(comment store.Comment) []byte {
return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
}
// parseRef gets parts of reference
func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
elems := strings.Split(string(val), "!!")
if len(elems) != 2 {
return "", "", errors.Errorf("invalid reference value %s", string(val))
}
return elems[0], elems[1], nil
}
backend/app/store/engine_old/bolt_accessor_test.go
-406
View File
@@ -1,406 +0,0 @@
package engine_old
import (
"fmt"
"os"
"testing"
"time"
bolt "github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
)
var testDb = "test-remark.db"
func TestBoltDB_CreateAndFind(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
assert.Equal(t, "user1", res[0].User.ID)
t.Log(res[0].ID)
_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
assert.NotNil(t, err)
assert.Equal(t, "key id-1 already in store", err.Error())
_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
assert.EqualError(t, err, `site "radio-t-bad" not found`)
assert.NoError(t, b.Close())
}
func TestBoltDB_CreateReadOnly(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
comment := store.Comment{
ID: "id-ro",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
err := b.SetReadOnly(comment.Locator, true)
require.Nil(t, err)
_, err = b.Create(comment)
assert.NotNil(t, err)
assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
err = b.SetReadOnly(comment.Locator, false)
require.Nil(t, err)
_, err = b.Create(comment)
assert.Nil(t, err)
}
func TestBoltDB_Get(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
assert.Nil(t, err)
assert.Equal(t, "some text2", comment.Text)
comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
assert.NotNil(t, err)
_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_Put(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment := res[0]
comment.Text = "abc 123"
comment.Score = 100
err = b.Put(loc, comment)
assert.Nil(t, err)
comment, err = b.Get(loc, res[0].ID)
assert.Nil(t, err)
assert.Equal(t, "abc 123", comment.Text)
assert.Equal(t, res[0].ID, comment.ID)
assert.Equal(t, 100, comment.Score)
err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
assert.EqualError(t, err, `site "bad" not found`)
err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
}
func TestBoltDB_Last(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
res, err := b.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
res, err = b.Last("radio-t", 1, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
_, err = b.Last("bad", 0, time.Time{})
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_LastSince(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
res, err := b.Last("radio-t", 0, ts)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
ts = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
res, err = b.Last("radio-t", 0, ts)
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
ts = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
res, err = b.Last("radio-t", 0, ts)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestBoltDB_Count(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 2, c)
c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c)
_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_List(t *testing.T) {
b, teardown := prep(t) // two comments for https://radio-t.com
defer teardown()
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := b.Create(comment)
assert.Nil(t, err)
ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
res, err := b.List("radio-t", 0, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
res)
res, err = b.List("radio-t", -1, -1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
res, err = b.List("radio-t", 1, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
res, err = b.List("radio-t", 1, 1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
_, err = b.List("bad", 1, 1)
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_Info(t *testing.T) {
b, teardown := prep(t) // two comments for https://radio-t.com
defer teardown()
ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := b.Create(comment)
assert.Nil(t, err)
r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
require.NotNil(t, err)
_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
require.NotNil(t, err)
err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
require.Nil(t, err)
r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
}
func TestBoltDB_GetForUser(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
res, err := b.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = b.User("radio-t", "user1", 1, 0)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = b.User("radio-t", "user1", 1, 1)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
_, err = b.User("bad", "user1", 1, 0)
assert.EqualError(t, err, `site "bad" not found`)
_, err = b.User("radio-t", "userZ", 1, 0)
assert.EqualError(t, err, `no comments for user userZ in store`)
}
func TestBoltDB_GetForUserPagination(t *testing.T) {
os.Remove(testDb)
b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
defer func() {
require.NoError(t, b.Close())
os.Remove(testDb)
}()
c := store.Comment{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
// write 200 comments
for i := 0; i < 200; i++ {
c.ID = fmt.Sprintf("id-%d", i)
c.Text = fmt.Sprintf("text #%d", i)
c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
_, err = b.Create(c)
require.Nil(t, err)
}
// get all comments
res, err := b.User("radio-t", "user1", 0, 0)
assert.Nil(t, err)
assert.Equal(t, 200, len(res))
assert.Equal(t, "id-199", res[0].ID)
// seek 0, 5 comments
res, err = b.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-199", res[0].ID)
assert.Equal(t, "id-195", res[4].ID)
// seek 10, 3 comments
res, err = b.User("radio-t", "user1", 3, 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, "id-189", res[0].ID)
assert.Equal(t, "id-187", res[2].ID)
// seek 195, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 195)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-4", res[0].ID)
assert.Equal(t, "id-0", res[4].ID)
// seek 255, ask 10 comments
res, err = b.User("radio-t", "user1", 10, 255)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestBoltDB_GetForUserCounter(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
count, err := b.UserCount("radio-t", "user1")
assert.Nil(t, err)
assert.Equal(t, 2, count)
_, err = b.UserCount("bad", "user1")
assert.EqualError(t, err, `site "bad" not found`)
_, err = b.UserCount("radio-t", "userZ")
assert.EqualError(t, err, `no comments for user userZ in store`)
}
func TestBoltDB_Ref(t *testing.T) {
b := BoltDB{}
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
res := b.makeRef(comment)
assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
assert.Nil(t, err)
assert.Equal(t, "https://radio-t.com/2", url)
assert.Equal(t, "12345", id)
_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
assert.NotNil(t, err)
}
func TestBoltDB_New(t *testing.T) {
_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
}
// makes new boltdb, put two records
func prep(t *testing.T) (b *BoltDB, teardown func()) {
os.Remove(testDb)
boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
assert.Nil(t, err)
b = boltStore
comment := store.Comment{
ID: "id-1",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err = b.Create(comment)
assert.Nil(t, err)
comment = store.Comment{
ID: "id-2",
Text: "some text2",
Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err = b.Create(comment)
assert.Nil(t, err)
teardown = func() {
require.NoError(t, b.Close())
os.Remove(testDb)
}
return b, teardown
}
backend/app/store/engine_old/bolt_admin.go
-339
View File
@@ -1,339 +0,0 @@
package engine_old
import (
"encoding/json"
"time"
bolt "github.com/coreos/bbolt"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
comment := store.Comment{}
if err = b.load(postBkt, commentID, &comment); err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted(mode)
if err = b.save(postBkt, commentID, comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
// delete from "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
if err = lastBkt.Delete([]byte(commentID)); err != nil {
return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
// DeleteAll removes all top-level buckets for given siteID
func (b *BoltDB) DeleteAll(siteID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
}
}
return nil
})
return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
}
// DeleteUser removes all comments for given user. Everything will be market as deleted
// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
func (b *BoltDB) DeleteUser(siteID string, userID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// get list of all comments outside of transaction loop
posts, err := b.List(siteID, 0, 0)
if err != nil {
return err
}
type commentInfo struct {
locator store.Locator
commentID string
}
// get list of commentID for all user's comment
comments := []commentInfo{}
for _, postInfo := range posts {
err = bdb.View(func(tx *bolt.Tx) error {
postsBkt := tx.Bucket([]byte(postsBucketName))
postBkt := postsBkt.Bucket([]byte(postInfo.URL))
err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
comment := store.Comment{}
if err = json.Unmarshal(commentVal, &comment); err != nil {
return errors.Wrap(err, "failed to unmarshal")
}
if comment.User.ID == userID {
comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
}
return nil
})
return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
})
if err != nil {
return err
}
}
log.Printf("[DEBUG] comments for removal=%d", len(comments))
// delete collected comments
for _, ci := range comments {
if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
return errors.Wrapf(err, "failed to delete comment %+v", ci)
}
}
// delete user bucket
err = bdb.Update(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
if usersBkt != nil {
if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
}
}
return nil
})
if err != nil {
return errors.Wrap(err, "can't delete user meta")
}
if len(comments) == 0 {
return errors.Errorf("unknown user %s", userID)
}
return err
}
// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
// block uses blocksBucketName with key=userID and val=TTL+now
func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
switch status {
case true:
val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
if ttl > 0 {
val = time.Now().Add(ttl).Format(tsNano)
}
if e := bucket.Put([]byte(userID), []byte(val)); e != nil {
return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
}
}
return nil
})
}
// IsBlocked checks if user blocked
func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
val := bucket.Get([]byte(userID))
if val == nil {
blocked = false
return nil
}
until, e := time.Parse(tsNano, string(val))
if e != nil {
blocked = false
return nil
}
blocked = time.Now().Before(until)
return nil
})
return blocked
}
// Blocked get lists of blocked users for given site
// bucket uses userID:
func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
if errParse != nil {
return errors.Wrap(errParse, "can't parse block ts")
}
if time.Now().Before(ts) {
// get user name from comment user section
userName := ""
userComments, errUser := b.User(siteID, string(k), 1, 0)
if errUser == nil && len(userComments) > 0 {
userName = userComments[0].User.Name
}
users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
}
return nil
})
})
return users, err
}
// SetReadOnly makes post read-only or reset the ro flag
func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(readonlyBucketName))
switch status {
case true:
if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
}
case false:
if e := bucket.Delete([]byte(locator.URL)); e != nil {
return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
}
}
return nil
})
}
// IsReadOnly checks if post in RO mode
func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
bdb, err := b.db(locator.SiteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(readonlyBucketName))
ro = bucket.Get([]byte(locator.URL)) != nil
return nil
})
return ro
}
// SetVerified makes user verified or reset the flag
func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(verifiedBucketName))
switch status {
case true:
if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to set verified status for %s", userID)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean verified status for %s", userID)
}
}
return nil
})
}
// IsVerified checks if user verified
func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(verifiedBucketName))
verified = bucket.Get([]byte(userID)) != nil
return nil
})
return verified
}
// Verified returns list of verified userIDs
func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(verifiedBucketName))
_ = usersBkt.ForEach(func(k, _ []byte) error {
ids = append(ids, string(k))
return nil
})
return nil
})
return ids, err
}
backend/app/store/engine_old/bolt_admin_test.go
-246
View File
@@ -1,246 +0,0 @@
package engine_old
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
)
func TestBoltAdmin_Delete(t *testing.T) {
b, teardown := prep(t)
defer teardown()
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
count, err := b.Count(loc)
require.NoError(t, err)
assert.Equal(t, 2, count, "count=2 initially")
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
count, err = b.Count(loc)
require.NoError(t, err)
assert.Equal(t, 1, count)
err = b.Delete(loc, "123456", store.SoftDelete)
assert.NotNil(t, err)
loc.SiteID = "bad"
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `site "bad" not found`)
loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
}
func TestBoltAdmin_DeleteHard(t *testing.T) {
b, teardown := prep(t)
defer teardown()
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID, store.HardDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
}
func TestBoltAdmin_DeleteAll(t *testing.T) {
b, teardown := prep(t)
defer teardown()
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.DeleteAll("radio-t")
assert.Nil(t, err)
comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
err = b.DeleteAll("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltAdmin_DeleteUser(t *testing.T) {
b, teardown := prep(t)
defer teardown()
err := b.DeleteUser("radio-t", "user1")
require.NoError(t, err)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
_, err = b.User("radio-t", "user1", 5, 0)
assert.EqualError(t, err, "no comments for user user1 in store")
comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
err = b.DeleteUser("radio-t-bad", "user1")
assert.EqualError(t, err, `site "radio-t-bad" not found`)
}
func TestBoltAdmin_BlockUser(t *testing.T) {
b, teardown := prep(t)
defer teardown()
assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0))
assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0))
assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
}
func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
b, teardown := prep(t)
defer teardown()
assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
time.Sleep(50 * time.Millisecond)
assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
}
func TestBoltAdmin_BlockList(t *testing.T) {
b, teardown := prep(t)
defer teardown()
assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0))
ids, err := b.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 2, len(ids))
assert.Equal(t, "user1", ids[0].ID)
assert.Equal(t, "user2", ids[1].ID)
t.Logf("%+v", ids)
time.Sleep(50 * time.Millisecond)
ids, err = b.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 1, len(ids))
assert.Equal(t, "user1", ids[0].ID)
_, err = b.Blocked("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltAdmin_ReadOnly(t *testing.T) {
b, teardown := prep(t)
defer teardown()
assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
}
func TestBoltAdmin_Verified(t *testing.T) {
b, teardown := prep(t)
defer teardown()
assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
assert.NoError(t, b.SetVerified("radio-t", "u1", true))
assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
assert.NoError(t, b.SetVerified("radio-t", "u1", false))
assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
assert.NoError(t, b.SetVerified("radio-t", "u1", true))
assert.NoError(t, b.SetVerified("radio-t", "u2", true))
assert.NoError(t, b.SetVerified("radio-t", "u3", false))
ids, err := b.Verified("radio-t")
assert.NoError(t, err)
assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
_, err = b.Verified("radio-t-bad")
assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
}
backend/app/store/engine_old/engine.go
-87
View File
@@ -1,87 +0,0 @@
// Package engine defines interfaces each supported storage should implement.
// Includes default implementation with boltdb
package engine_old
import (
"sort"
"strings"
"time"
"github.com/umputun/remark/backend/app/store"
)
// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
// Interface defines methods provided by low-level storage engine
type Interface interface {
Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id
Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only
Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator
Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time
User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
UserCount(siteID, userID string) (int, error) // comments count by user
Count(locator store.Locator) (int, error) // number of comments for the post
List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
DeleteAll(siteID string) error // delete all data from site
DeleteUser(siteID string, userID string) error // remove all comments from user
SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent)
IsBlocked(siteID string, userID string) bool // check if user blocked
Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag
IsReadOnly(locator store.Locator) bool // check if post read-only
SetVerified(siteID string, userID string, status bool) error // set/reset verified flag
IsVerified(siteID string, userID string) bool // check verified status
Verified(siteID string) ([]string, error) // list of verified user ids
Close() error // close/stop engine
}
const (
// limits
lastLimit = 1000
userLimit = 500
)
// SortComments is for engines can't sort data internally
func SortComments(comments []store.Comment, sortFld string) []store.Comment {
sort.Slice(comments, func(i, j int) bool {
switch sortFld {
case "+time", "-time", "time", "+active", "-active", "active":
if strings.HasPrefix(sortFld, "-") {
return comments[i].Timestamp.After(comments[j].Timestamp)
}
return comments[i].Timestamp.Before(comments[j].Timestamp)
case "+score", "-score", "score":
if strings.HasPrefix(sortFld, "-") {
if comments[i].Score == comments[j].Score {
return comments[i].Timestamp.Before(comments[j].Timestamp)
}
return comments[i].Score > comments[j].Score
}
if comments[i].Score == comments[j].Score {
return comments[i].Timestamp.Before(comments[j].Timestamp)
}
return comments[i].Score < comments[j].Score
case "+controversy", "-controversy", "controversy":
if strings.HasPrefix(sortFld, "-") {
if comments[i].Controversy == comments[j].Controversy {
return comments[i].Timestamp.Before(comments[j].Timestamp)
}
return comments[i].Controversy > comments[j].Controversy
}
if comments[i].Controversy == comments[j].Controversy {
return comments[i].Timestamp.Before(comments[j].Timestamp)
}
return comments[i].Controversy < comments[j].Controversy
default:
return comments[i].Timestamp.Before(comments[j].Timestamp)
}
})
return comments
}
backend/app/store/engine_old/engine_mock.go
-408
View File
@@ -1,408 +0,0 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package engine_old
import mock "github.com/stretchr/testify/mock"
import store "github.com/umputun/remark/backend/app/store"
import time "time"
// MockInterface is an autogenerated mock type for the Interface type
type MockInterface struct {
mock.Mock
}
// Blocked provides a mock function with given fields: siteID
func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) {
ret := _m.Called(siteID)
var r0 []store.BlockedUser
if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok {
r0 = rf(siteID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.BlockedUser)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(siteID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Close provides a mock function with given fields:
func (_m *MockInterface) Close() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Count provides a mock function with given fields: locator
func (_m *MockInterface) Count(locator store.Locator) (int, error) {
ret := _m.Called(locator)
var r0 int
if rf, ok := ret.Get(0).(func(store.Locator) int); ok {
r0 = rf(locator)
} else {
r0 = ret.Get(0).(int)
}
var r1 error
if rf, ok := ret.Get(1).(func(store.Locator) error); ok {
r1 = rf(locator)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Create provides a mock function with given fields: comment
func (_m *MockInterface) Create(comment store.Comment) (string, error) {
ret := _m.Called(comment)
var r0 string
if rf, ok := ret.Get(0).(func(store.Comment) string); ok {
r0 = rf(comment)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(store.Comment) error); ok {
r1 = rf(comment)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Delete provides a mock function with given fields: locator, commentID, mode
func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
ret := _m.Called(locator, commentID, mode)
var r0 error
if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok {
r0 = rf(locator, commentID, mode)
} else {
r0 = ret.Error(0)
}
return r0
}
// DeleteAll provides a mock function with given fields: siteID
func (_m *MockInterface) DeleteAll(siteID string) error {
ret := _m.Called(siteID)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(siteID)
} else {
r0 = ret.Error(0)
}
return r0
}
// DeleteUser provides a mock function with given fields: siteID, userID
func (_m *MockInterface) DeleteUser(siteID string, userID string) error {
ret := _m.Called(siteID, userID)
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(siteID, userID)
} else {
r0 = ret.Error(0)
}
return r0
}
// Find provides a mock function with given fields: locator, sort
func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) {
ret := _m.Called(locator, sort)
var r0 []store.Comment
if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok {
r0 = rf(locator, sort)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.Comment)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok {
r1 = rf(locator, sort)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Get provides a mock function with given fields: locator, commentID
func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) {
ret := _m.Called(locator, commentID)
var r0 store.Comment
if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok {
r0 = rf(locator, commentID)
} else {
r0 = ret.Get(0).(store.Comment)
}
var r1 error
if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok {
r1 = rf(locator, commentID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Info provides a mock function with given fields: locator, readonlyAge
func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
ret := _m.Called(locator, readonlyAge)
var r0 store.PostInfo
if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok {
r0 = rf(locator, readonlyAge)
} else {
r0 = ret.Get(0).(store.PostInfo)
}
var r1 error
if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok {
r1 = rf(locator, readonlyAge)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// IsBlocked provides a mock function with given fields: siteID, userID
func (_m *MockInterface) IsBlocked(siteID string, userID string) bool {
ret := _m.Called(siteID, userID)
var r0 bool
if rf, ok := ret.Get(0).(func(string, string) bool); ok {
r0 = rf(siteID, userID)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// IsReadOnly provides a mock function with given fields: locator
func (_m *MockInterface) IsReadOnly(locator store.Locator) bool {
ret := _m.Called(locator)
var r0 bool
if rf, ok := ret.Get(0).(func(store.Locator) bool); ok {
r0 = rf(locator)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// IsVerified provides a mock function with given fields: siteID, userID
func (_m *MockInterface) IsVerified(siteID string, userID string) bool {
ret := _m.Called(siteID, userID)
var r0 bool
if rf, ok := ret.Get(0).(func(string, string) bool); ok {
r0 = rf(siteID, userID)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// Last provides a mock function with given fields: siteID, limit, since
func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) {
ret := _m.Called(siteID, limit, since)
var r0 []store.Comment
if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok {
r0 = rf(siteID, limit, since)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.Comment)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok {
r1 = rf(siteID, limit, since)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// List provides a mock function with given fields: siteID, limit, skip
func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
ret := _m.Called(siteID, limit, skip)
var r0 []store.PostInfo
if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok {
r0 = rf(siteID, limit, skip)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.PostInfo)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, int, int) error); ok {
r1 = rf(siteID, limit, skip)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Put provides a mock function with given fields: locator, comment
func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error {
ret := _m.Called(locator, comment)
var r0 error
if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok {
r0 = rf(locator, comment)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetBlock provides a mock function with given fields: siteID, userID, status, ttl
func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
ret := _m.Called(siteID, userID, status, ttl)
var r0 error
if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok {
r0 = rf(siteID, userID, status, ttl)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetReadOnly provides a mock function with given fields: locator, status
func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error {
ret := _m.Called(locator, status)
var r0 error
if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok {
r0 = rf(locator, status)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetVerified provides a mock function with given fields: siteID, userID, status
func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error {
ret := _m.Called(siteID, userID, status)
var r0 error
if rf, ok := ret.Get(0).(func(string, string, bool) error); ok {
r0 = rf(siteID, userID, status)
} else {
r0 = ret.Error(0)
}
return r0
}
// User provides a mock function with given fields: siteID, userID, limit, skip
func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) {
ret := _m.Called(siteID, userID, limit, skip)
var r0 []store.Comment
if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok {
r0 = rf(siteID, userID, limit, skip)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.Comment)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok {
r1 = rf(siteID, userID, limit, skip)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// UserCount provides a mock function with given fields: siteID, userID
func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) {
ret := _m.Called(siteID, userID)
var r0 int
if rf, ok := ret.Get(0).(func(string, string) int); ok {
r0 = rf(siteID, userID)
} else {
r0 = ret.Get(0).(int)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(siteID, userID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Verified provides a mock function with given fields: siteID
func (_m *MockInterface) Verified(siteID string) ([]string, error) {
ret := _m.Called(siteID)
var r0 []string
if rf, ok := ret.Get(0).(func(string) []string); ok {
r0 = rf(siteID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(siteID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
backend/app/store/engine_old/engine_test.go
-55
View File
@@ -1,55 +0,0 @@
package engine_old
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store"
)
func TestEngine_sortComments(t *testing.T) {
cc := []store.Comment{
{ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
{ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
{ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
{ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
}
SortComments(cc, "+time")
assert.Equal(t, "1", cc[0].ID)
assert.Equal(t, "2", cc[1].ID)
assert.Equal(t, "3", cc[2].ID)
assert.Equal(t, "4", cc[3].ID)
SortComments(cc, "-time")
assert.Equal(t, "4", cc[0].ID)
assert.Equal(t, "3", cc[1].ID)
assert.Equal(t, "2", cc[2].ID)
assert.Equal(t, "1", cc[3].ID)
SortComments(cc, "score")
assert.Equal(t, "2", cc[0].ID)
assert.Equal(t, "1", cc[1].ID)
assert.Equal(t, "3", cc[2].ID)
assert.Equal(t, "4", cc[3].ID)
SortComments(cc, "-score")
assert.Equal(t, "3", cc[0].ID)
assert.Equal(t, "4", cc[1].ID)
assert.Equal(t, "1", cc[2].ID)
assert.Equal(t, "2", cc[3].ID)
SortComments(cc, "controversy")
assert.Equal(t, "1", cc[0].ID)
assert.Equal(t, "4", cc[1].ID)
assert.Equal(t, "2", cc[2].ID)
assert.Equal(t, "3", cc[3].ID)
SortComments(cc, "-controversy")
assert.Equal(t, "3", cc[0].ID)
assert.Equal(t, "2", cc[1].ID)
assert.Equal(t, "1", cc[2].ID)
assert.Equal(t, "4", cc[3].ID)
}
backend/app/store/engine_old/mongo.go
-380
View File
@@ -1,380 +0,0 @@
package engine_old
import (
"time"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// Mongo implements engine interface
type Mongo struct {
conn *mongo.Connection
postWriter mongo.BufferedWriter
}
const (
mongoPosts = "posts"
mongoMetaPosts = "meta_posts"
mongoMetaUsers = "meta_users"
)
type metaPost struct {
ID string `bson:"_id"` // url
SiteID string `bson:"site"`
ReadOnly bool `bson:"read_only"`
}
type metaUser struct {
ID string `bson:"_id"` // user_id
SiteID string `bson:"site"`
Verified bool `bson:"verified"`
Blocked bool `bson:"blocked"`
BlockedUntil time.Time `bson:"blocked_until"`
}
// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached.
// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
result := Mongo{conn: conn, postWriter: writer}
err := result.prepare()
return &result, errors.Wrap(err, "failed to prepare mongo")
}
// Create new comment, write can be buffered and delayed.
func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
// err = m.postWriter.Write(comment)
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
return coll.Insert(&comment)
})
return comment.ID, err
}
// Find returns all comments for post and sorts results
func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
comments = []store.Comment{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
return coll.Find(query).Sort(sortFld).All(&comments)
})
return comments, err
}
// Get returns comment for locator.URL and commentID string
func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
return coll.Find(query).One(&comment)
})
return comment, err
}
// Put updates comment for locator.URL with mutable part of comment
func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
bson.M{"$set": bson.M{
"text": comment.Text,
"orig": comment.Orig,
"score": comment.Score,
"votes": comment.Votes,
"pin": comment.Pin,
"deleted": comment.Deleted,
}})
})
}
// Last returns up to max last comments for given siteID
func (m *Mongo) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
comments = []store.Comment{}
if max > lastLimit || max == 0 {
max = lastLimit
}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": siteID, "delete": false}
if !since.IsZero() {
query["time"] = bson.M{"$gt": since}
}
return coll.Find(query).Sort("-time").Limit(max).All(&comments)
})
return comments, err
}
// Count returns number of comments for locator
func (m *Mongo) Count(locator store.Locator) (count int, err error) {
e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
count, err = coll.Find(query).Count()
return err
})
return count, e
}
// List returns list of all commented posts with counters
func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
list = []store.PostInfo{}
if limit <= 0 {
limit = 1000
}
if skip < 0 {
skip = 0
}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
pipeline := coll.Pipe([]bson.M{
{"$match": bson.M{"locator.site": siteID}},
{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
{"$skip": skip},
{"$limit": limit},
})
return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
})
return list, errors.Wrap(err, "can't get list")
}
// Info returns time range and count for locator
func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
list := []store.PostInfo{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
pipeline := coll.Pipe([]bson.M{
{"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
})
return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
})
if err != nil {
return info, err
}
if len(list) == 0 {
return info, errors.Errorf("can't load info for %s", locator.URL)
}
info = list[0]
// set read-only from age and manual bucket
info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
if m.IsReadOnly(locator) {
info.ReadOnly = true
}
return info, nil
}
// User extracts all comments for given site and given userID
func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
comments = []store.Comment{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": siteID, "user.id": userID}
return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
})
return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
}
// UserCount returns number of comments for user
func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
var e error
count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
return e
})
return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
}
// SetReadOnly makes post read-only or reset the ro flag
func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
return e
})
}
// IsReadOnly checks if post in RO
func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
meta := metaPost{}
err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
})
return err == nil && meta.ReadOnly
}
// SetVerified makes user verified or reset the flag
func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
return e
})
}
// IsVerified checks if user verified
func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
meta := metaUser{}
err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
})
return err == nil && meta.Verified
}
// Verified returns list of verified user IDs
func (m *Mongo) Verified(siteID string) (ids []string, err error) {
metas := []metaUser{}
err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
})
if err != nil {
return nil, err
}
for _, meta := range metas {
ids = append(ids, meta.ID)
}
return ids, nil
}
// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
// block uses blocksBucketName with key=userID and val=TTL+now
func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
until := time.Time{}
if status {
until = time.Now().AddDate(100, 0, 0) // permanent is 50year
if ttl > 0 {
until = time.Now().Add(ttl)
}
}
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
return errors.Wrapf(e, "failed to set block for %s", userID)
})
}
// IsBlocked checks if user blocked
func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
meta := metaUser{}
err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
})
return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
}
// Blocked get lists of blocked users for given site
func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
metas := []metaUser{}
err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID,
"blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
})
if err != nil {
return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
}
for _, mu := range metas {
blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
blockedUser.Name = ucc[0].User.Name
}
users = append(users, blockedUser)
}
return users, nil
}
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
comment := store.Comment{}
err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
if e != nil {
return e
}
comment.SetDeleted(mode)
return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
})
return errors.Wrapf(err, "can't delete %s", commentID)
}
// DeleteAll removes all info about siteID
func (m *Mongo) DeleteAll(siteID string) error {
err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
_, e := coll.RemoveAll(bson.M{"locator.site": siteID})
return e
})
return errors.Wrapf(err, "can't delete site %s", siteID)
}
// DeleteUser removes all comments for given user. Everything will be market as deleted
// and user name and userID will be changed to "deleted".
func (m *Mongo) DeleteUser(siteID string, userID string) error {
comments := []store.Comment{}
return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
if e != nil {
return e
}
for _, c := range comments {
if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
return e
}
}
return nil
})
}
// Close boltdb store
func (m *Mongo) Close() error {
if m.postWriter != nil {
return m.postWriter.Close()
}
return nil
}
// prepare collections with all indexes
func (m *Mongo) prepare() error {
errs := new(multierror.Error)
e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
})
if e != nil {
return e
}
e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
})
if e != nil {
return e
}
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
})
}
func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
if limit <= 0 {
limit = 1000
}
if skip < 0 {
skip = 0
}
return q.Skip(skip).Limit(limit)
}
backend/app/store/engine_old/mongo_test.go
-604
View File
@@ -1,604 +0,0 @@
package engine_old
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
)
func TestMongo_CreateAndFind(t *testing.T) {
var m Interface
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
require.Equal(t, 2, len(res))
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
assert.Equal(t, "user1", res[0].User.ID)
t.Log(res[0].ID)
_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
assert.NotNil(t, err, "reject dup")
id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
assert.Nil(t, err)
assert.Equal(t, "id-3", id)
res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
assert.Nil(t, err)
require.Equal(t, 1, len(res))
assert.NoError(t, m.Close())
}
func TestMongo_Get(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
assert.Nil(t, err)
assert.Equal(t, "some text2", comment.Text)
comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
assert.NotNil(t, err, "not found")
}
func TestMongo_Put(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment := res[0]
comment.Text = "abc 123"
comment.Score = 100
err = m.Put(loc, comment)
assert.Nil(t, err)
comment, err = m.Get(loc, res[0].ID)
assert.Nil(t, err)
assert.Equal(t, "abc 123", comment.Text)
assert.Equal(t, res[0].ID, comment.ID)
assert.Equal(t, 100, comment.Score)
err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
assert.EqualError(t, err, `not found`)
err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
assert.EqualError(t, err, `not found`)
}
func TestMongo_Last(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local))
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local))
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
res, err = m.Last("radio-t", 1, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
}
func TestMongo_Count(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 2, c)
c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c)
}
func TestMongo_List(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := m.Create(comment)
assert.Nil(t, err)
ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
res, err := m.List("radio-t", 0, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
res)
res, err = m.List("radio-t", -1, -1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
res, err = m.List("radio-t", 1, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
res, err = m.List("radio-t", 1, 1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
res, err = m.List("bad", 1, 1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{}, res)
}
func TestMongo_Info(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := m.Create(comment)
assert.Nil(t, err)
r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
_, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
require.NotNil(t, err)
_, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
require.NotNil(t, err)
err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
require.Nil(t, err)
r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
}
func TestMongo_ReadOnly(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
}
func TestMongo_Verified(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
assert.NoError(t, m.SetVerified("radio-t", "u1", true))
assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
assert.NoError(t, m.SetVerified("radio-t", "u1", false))
assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
assert.NoError(t, m.SetVerified("radio-t", "u1", true))
assert.NoError(t, m.SetVerified("radio-t", "u2", true))
assert.NoError(t, m.SetVerified("radio-t", "u3", false))
ids, err := m.Verified("radio-t")
assert.NoError(t, err)
assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
ids, err = m.Verified("radio-t-bad")
assert.NoError(t, err)
assert.Equal(t, 0, len(ids))
}
func TestMongo_GetForUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = m.User("radio-t", "user1", 1, 0)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = m.User("radio-t", "user1", 1, 1)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
res, err = m.User("bad", "user1", 1, 0)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestMongo_GetForUserPagination(t *testing.T) {
m, skip := prepMongo(t, false)
if skip {
return
}
c := store.Comment{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
// write 200 comments
for i := 0; i < 200; i++ {
c.ID = fmt.Sprintf("id-%d", i)
c.Text = fmt.Sprintf("text #%d", i)
c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
_, err := m.Create(c)
require.Nil(t, err, c.ID)
}
// get all comments
res, err := m.User("radio-t", "user1", 0, 0)
assert.Nil(t, err)
assert.Equal(t, 200, len(res))
assert.Equal(t, "id-199", res[0].ID)
// seek 0, 5 comments
res, err = m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-199", res[0].ID)
assert.Equal(t, "id-195", res[4].ID)
// seek 10, 3 comments
res, err = m.User("radio-t", "user1", 3, 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, "id-189", res[0].ID)
assert.Equal(t, "id-187", res[2].ID)
// seek 195, ask 10 comments
res, err = m.User("radio-t", "user1", 10, 195)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-4", res[0].ID)
assert.Equal(t, "id-0", res[4].ID)
// seek 255, ask 10 comments
res, err = m.User("radio-t", "user1", 10, 255)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestMongo_BlockUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
}
func TestMongo_BlockUserWithTTL(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
time.Sleep(500 * time.Millisecond)
assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
}
func TestMongo_GetForUserCounter(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
count, err := m.UserCount("radio-t", "user1")
assert.Nil(t, err)
assert.Equal(t, 2, count)
count, err = m.UserCount("bad", "user1")
assert.Nil(t, err)
assert.Equal(t, 0, count)
}
func TestMongo_BlockList(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
ids, err := m.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 2, len(ids))
assert.Equal(t, "user1", ids[0].ID)
assert.Equal(t, "user2", ids[1].ID)
t.Logf("%+v", ids)
time.Sleep(500 * time.Millisecond)
ids, err = m.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 1, len(ids))
assert.Equal(t, "user1", ids[0].ID)
ids, err = m.Blocked("bad")
assert.NoError(t, err)
assert.Equal(t, 0, len(ids))
}
func TestMongo_Delete(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.Nil(t, err)
res, err = m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
err = m.Delete(loc, "123456", store.SoftDelete)
assert.NotNil(t, err)
loc.SiteID = "bad"
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `can't delete id-1: not found`)
loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `can't delete id-1: not found`)
}
func TestMongo_DeleteHard(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.Delete(loc, res[0].ID, store.HardDelete)
assert.Nil(t, err)
res, err = m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
}
func TestMongo_DeleteAll(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.DeleteAll("radio-t")
assert.Nil(t, err)
comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
}
func TestMongo_DeleteUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
err := m.DeleteUser("radio-t", "user1")
require.NoError(t, err)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
cc, err := m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err, "no comments for user user1 in store")
assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
}
func TestMongo_Parallel(t *testing.T) {
var m Interface
var skip bool
m, skip = prepMongoBuffered(t) // buffered engine, no comments
if skip {
return
}
go func() {
for i := 0; i < 100; i++ {
_, err := m.Create(store.Comment{
ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
require.Nil(t, err)
time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
}
}()
for {
time.Sleep(10 * time.Millisecond)
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
if len(res) == 100 {
break
}
}
}
func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
mongo.RemoveTestCollection(t, conn)
m, err := NewMongo(conn, 1, 0*time.Microsecond)
require.Nil(t, err)
mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
comment := store.Comment{
ID: "id-1",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
if writeRecs {
_, err = m.Create(comment)
assert.Nil(t, err)
}
comment = store.Comment{
ID: "id-2",
Text: "some text2",
Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
if writeRecs {
_, err = m.Create(comment)
assert.Nil(t, err)
}
return m, false
}
func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
mongo.RemoveTestCollection(t, conn)
m, err := NewMongo(conn, 10, 10*time.Millisecond)
mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
require.Nil(t, err)
return m, false
}
backend/app/store/remote/client.go
+1 -1
View File
@@ -27,7 +27,7 @@ func (r *Client) Call(method string, args ...interface{}) (*Response, error) {
var err error
switch {
case args == nil || len(args) == 0:
case len(args) == 0:
b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)})
if err != nil {
return nil, errors.Wrapf(err, "marshaling failed for %s", method)
backend/go.mod
+1 -2
View File
@@ -11,14 +11,13 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/didip/tollbooth v4.0.0+incompatible
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
github.com/go-chi/chi v4.0.2+incompatible
github.com/go-chi/cors v1.0.0
github.com/go-chi/render v1.0.1
github.com/go-pkgz/auth v0.5.2
github.com/go-pkgz/lcw v0.3.1
github.com/go-pkgz/lgr v0.6.2
github.com/go-pkgz/mongo v1.1.2
github.com/go-pkgz/mongo v1.1.2 // indirect
github.com/go-pkgz/repeater v1.1.2
github.com/go-pkgz/rest v1.4.1
github.com/go-pkgz/syncs v1.1.1