diff --git a/.dockerignore b/.dockerignore index 37da97c5..c016742c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,4 +20,5 @@ debug.test *.prof *.test remark42 -/backend/var/ \ No newline at end of file +/backend/var/ +compose-private-backend.yml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3eb5f67e..f6821858 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ debug.test remark42 /bin/ /backend/var/ +compose-private-backend.yml diff --git a/.travis.yml b/.travis.yml index 01016812..9cb6efc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ install: - docker-compose --version script: - - docker run -d --name=mongo mongo:3.6 && sleep 3 - - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) - - echo "running mongo on $MONGO_TEST" - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN --build-arg CI=$CI @@ -19,7 +16,4 @@ script: --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG --build-arg TRAVIS_TAG=$TRAVIS_TAG - --build-arg MONGO_TEST=$MONGO_TEST . - - docker rm -f mongo - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6724f223..63582ca5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ ARG DRONE_BRANCH ARG DRONE_PULL_REQUEST ARG SKIP_BACKEND_TEST -ARG MONGO_TEST ADD backend /build/backend ADD .git /build/.git @@ -29,18 +28,14 @@ ENV GOFLAGS="-mod=vendor" # run tests RUN \ - if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ cd app && \ if [ -z "$SKIP_BACKEND_TEST" ] ; then \ - go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \ + go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \ cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \ else echo "skip backend test" ; fi -RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts - # linters RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ - if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \ --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \ --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \ diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index e0075113..b8676483 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -16,7 +16,6 @@ import ( type AvatarCommand struct { AvatarSrc AvatarGroup `group:"src" namespace:"src"` AvatarDst AvatarGroup `group:"dst" namespace:"dst"` - // Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` migrator AvatarMigrator CommonOpts diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go index 203f246c..e31a53b8 100644 --- a/backend/app/cmd/avatar_test.go +++ b/backend/app/cmd/avatar_test.go @@ -13,13 +13,6 @@ import ( func TestAvatar_Execute(t *testing.T) { - mongoURL := os.Getenv("MONGO_TEST") - if mongoURL == "" { - mongoURL = "mongodb://localhost:27017/test" - } - if mongoURL == "skip" { - t.Skip("skip mongo app test") - } defer os.RemoveAll("/tmp/ava-test") // from fs to bolt @@ -37,7 +30,7 @@ func TestAvatar_Execute(t *testing.T) { cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", - "--dst.bolt.file=/tmp/ava-test.db"}) + "--dst.bolt.file=/tmp/ava-test2.db"}) require.NoError(t, err) err = cmd.Execute(nil) assert.Error(t, err, "failed blah") diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 6976593c..40aeb435 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -98,7 +98,7 @@ type StoreGroup struct { // ImageGroup defines options group for store pictures type ImageGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"` Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"` @@ -114,7 +114,7 @@ type ImageGroup struct { // AvatarGroup defines options group for avatar params type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` @@ -126,7 +126,7 @@ type AvatarGroup struct { // CacheGroup defines options group for cache params type CacheGroup struct { - Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"` + Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` Max struct { Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` @@ -176,7 +176,7 @@ type StreamGroup struct { // RemoteGroup defines options for remote modules (plugins) type RemoteGroup struct { API string `long:"api" env:"API" description:"remote extension api url"` - TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"` + TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"` AuthUser string `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"` AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"` } diff --git a/backend/app/store/engine_old/bolt_accessor.go b/backend/app/store/engine_old/bolt_accessor.go deleted file mode 100644 index f48c75da..00000000 --- a/backend/app/store/engine_old/bolt_accessor.go +++ /dev/null @@ -1,550 +0,0 @@ -package engine_old - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "time" - - bolt "github.com/coreos/bbolt" - log "github.com/go-pkgz/lgr" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe. -// there are 5 types of top-level buckets: -// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment -// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID -// value is not full comment but a reference combined from post-url+commentID -// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value -// is a nested bucket named userID with kv as ts:reference -// - blocking info sits in "block" bucket. Key is userID, value - ts -// - counts per post to keep number of comments. Key is post url, value - count -// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts -type BoltDB struct { - dbs map[string]*bolt.DB -} - -const ( - // top level buckets - postsBucketName = "posts" - lastBucketName = "last" - userBucketName = "users" - blocksBucketName = "block" - infoBucketName = "info" - readonlyBucketName = "readonly" - verifiedBucketName = "verified" - - tsNano = "2006-01-02T15:04:05.000000000Z07:00" -) - -// BoltSite defines single site param -type BoltSite struct { - FileName string // full path to boltdb - SiteID string // ID to access given site -} - -// NewBoltDB makes persistent boltdb-based store -func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { - log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options) - result := BoltDB{dbs: make(map[string]*bolt.DB)} - for _, site := range sites { - db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second} - if err != nil { - return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName) - } - - // make top-level buckets - topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, - infoBucketName, readonlyBucketName, verifiedBucketName} - err = db.Update(func(tx *bolt.Tx) error { - for _, bktName := range topBuckets { - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(e, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - if err != nil { - return nil, errors.Wrap(err, "failed to create top level bucket)") - } - - result.dbs[site.SiteID] = db - log.Printf("[DEBUG] bolt store created for %s", site.SiteID) - } - return &result, nil -} - -// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket -func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { - - bdb, err := b.db(comment.Locator.SiteID) - if err != nil { - return "", err - } - - if b.IsReadOnly(comment.Locator) { - return "", errors.Errorf("post %s is read-only", comment.Locator.URL) - } - - err = bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.makePostBucket(tx, comment.Locator.URL) - if e != nil { - return e - } - - // check if key already in store, reject doubles - if postBkt.Get([]byte(comment.ID)) != nil { - return errors.Errorf("key %s already in store", comment.ID) - } - - // serialize comment to json []byte for bolt and save - if e = b.save(postBkt, comment.ID, comment); e != nil { - return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL) - } - - ref := b.makeRef(comment) - - // add reference to comment to "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - commentTs := []byte(comment.Timestamp.Format(tsNano)) - e = lastBkt.Put(commentTs, ref) - if e != nil { - return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName) - } - - // add reference to commentID to "users" bucket - userBkt, e := b.getUserBucket(tx, comment.User.ID) - if e != nil { - return errors.Wrapf(e, "can't get bucket %s", comment.User.ID) - } - // put into individual user's bucket with ts as a key - if e = userBkt.Put(commentTs, ref); e != nil { - return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID) - } - - // set info with the count for post url - if _, e = b.setInfo(tx, comment); e != nil { - return errors.Wrapf(e, "failed to set info for %s", comment.Locator) - } - return nil - }) - - return comment.ID, err -} - -// Find returns all comments for post and sorts results -func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { - comments = []store.Comment{} - - bdb, err := b.db(locator.SiteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - return bucket.ForEach(func(k, v []byte) error { - comment := store.Comment{} - if e = json.Unmarshal(v, &comment); e != nil { - return errors.Wrap(e, "failed to unmarshal") - } - comments = append(comments, comment) - return nil - }) - }) - - comments = SortComments(comments, sortFld) - return comments, err -} - -// Last returns up to max last comments for given siteID -func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) { - - comments = []store.Comment{} - - if max > lastLimit || max == 0 { - max = lastLimit - } - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - lastBkt := tx.Bucket([]byte(lastBucketName)) - c := lastBkt.Cursor() - - for k, v := c.Last(); k != nil; k, v = c.Prev() { - - if !since.IsZero() { - // stop if reached "since" ts - tsSince := []byte(since.Format(tsNano)) - if bytes.Compare(k, tsSince) <= 0 { - break - } - } - url, commentID, e := b.parseRef(v) - if e != nil { - return e - } - postBkt, e := b.getPostBucket(tx, url) - if e != nil { - return e - } - - comment := store.Comment{} - if e = b.load(postBkt, commentID, &comment); e != nil { - log.Printf("[WARN] can't load comment for %s from store %s", commentID, url) - continue - } - if comment.Deleted { - continue - } - comments = append(comments, comment) - if len(comments) >= max { - break - } - } - return nil - }) - - return comments, err -} - -// Count returns number of comments for locator -func (b *BoltDB) Count(locator store.Locator) (count int, err error) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return 0, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - var e error - count, e = b.count(tx, locator.URL, 0) - return e - }) - - return count, err -} - -// List returns list of all commented posts with counters -// uses count bucket to get number of comments -func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - postsBkt := tx.Bucket([]byte(postsBucketName)) - - c := postsBkt.Cursor() - n := 0 - for k, _ := c.Last(); k != nil; k, _ = c.Prev() { - n++ - if skip > 0 && n <= skip { - continue - } - postURL := string(k) - infoBkt := tx.Bucket([]byte(infoBucketName)) - info := store.PostInfo{} - if e := b.load(infoBkt, postURL, &info); e != nil { - return errors.Wrapf(e, "can't load info for %s", postURL) - } - list = append(list, info) - if limit > 0 && len(list) >= limit { - break - } - } - return nil - }) - - return list, err -} - -// Info returns time range and count for locator -func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) { - bdb, err := b.db(locator.SiteID) - if err != nil { - return store.PostInfo{}, err - } - - info := store.PostInfo{} - err = bdb.View(func(tx *bolt.Tx) error { - infoBkt := tx.Bucket([]byte(infoBucketName)) - if e := b.load(infoBkt, locator.URL, &info); e != nil { - return errors.Wrapf(e, "can't load info for %s", locator.URL) - } - return nil - }) - - // set read-only from age and manual bucket - info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now()) - if b.IsReadOnly(locator) { - info.ReadOnly = true - } - return info, err -} - -// User extracts all comments for given site and given userID -// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref -func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { - - comments = []store.Comment{} - commentRefs := []string{} - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - if limit == 0 || limit > userLimit { - limit = userLimit - } - - // get list of references to comments - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt := usersBkt.Bucket([]byte(userID)) - if userIDBkt == nil { - return errors.Errorf("no comments for user %s in store", userID) - } - - c := userIDBkt.Cursor() - skipComments := 0 - for k, v := c.Last(); k != nil; k, v = c.Prev() { - if len(commentRefs) >= limit { - break - } - if skip > 0 && skipComments < skip { - skipComments++ - continue - } - commentRefs = append(commentRefs, string(v)) - } - return nil - }) - - if err != nil { - return comments, err - } - - // retrieve comments for refs - for _, v := range commentRefs { - url, commentID, errParse := b.parseRef([]byte(v)) - if errParse != nil { - return comments, errors.Wrapf(errParse, "can't parse reference %s", v) - } - if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil { - comments = append(comments, c) - } - } - - return comments, err -} - -// UserCount returns number of comments for user -func (b *BoltDB) UserCount(siteID, userID string) (int, error) { - bdb, err := b.db(siteID) - if err != nil { - return 0, err - } - count := 0 - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt := usersBkt.Bucket([]byte(userID)) - if userIDBkt == nil { - return errors.Errorf("no comments for user %s in store", userID) - } - stats := userIDBkt.Stats() - count = stats.KeyN - return nil - }) - return count, err -} - -// Get returns comment for locator.URL and commentID string -func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return comment, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - return b.load(bucket, commentID, &comment) - }) - return comment, err -} - -// Put updates comment for locator.URL with mutable part of comment -func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error { - - if curComment, err := b.Get(locator, comment.ID); err == nil { - // preserve immutable fields - comment.ParentID = curComment.ParentID - comment.Locator = curComment.Locator - comment.Timestamp = curComment.Timestamp - comment.User = curComment.User - } - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - return b.save(bucket, comment.ID, comment) - }) -} - -// Close boltdb store -func (b *BoltDB) Close() error { - errs := new(multierror.Error) - for site, db := range b.dbs { - err := errors.Wrapf(db.Close(), "can't close site %s", site) - errs = multierror.Append(errs, err) - } - return errs.ErrorOrNil() -} - -// getPostBucket return bucket with all comments for postURL -func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { - postsBkt := tx.Bucket([]byte(postsBucketName)) - if postsBkt == nil { - return nil, errors.Errorf("no bucket %s", postsBucketName) - } - res := postsBkt.Bucket([]byte(postURL)) - if res == nil { - return nil, errors.Errorf("no bucket %s in store", postURL) - } - return res, nil -} - -// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post. -func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { - postsBkt := tx.Bucket([]byte(postsBucketName)) - if postsBkt == nil { - return nil, errors.Errorf("no bucket %s", postsBucketName) - } - res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL)) - if err != nil { - return nil, errors.Wrapf(err, "no bucket %s in store", postURL) - } - return res, nil -} - -func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID - if e != nil { - return nil, errors.Wrapf(e, "can't get bucket %s", userID) - } - return userIDBkt, nil -} - -// save marshaled value to key for bucket. Should run in update tx -func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) { - if value == nil { - return errors.Errorf("can't save nil value for %s", key) - } - jdata, jerr := json.Marshal(value) - if jerr != nil { - return errors.Wrap(jerr, "can't marshal comment") - } - if err = bkt.Put([]byte(key), jdata); err != nil { - return errors.Wrapf(err, "failed to save key %s", key) - } - return nil -} - -// load and unmarshal json value by key from bucket. Should run in view tx -func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error { - value := bkt.Get([]byte(key)) - if value == nil { - return errors.Errorf("no value for %s", key) - } - - if err := json.Unmarshal(value, &res); err != nil { - return errors.Wrap(err, "failed to unmarshal") - } - return nil -} - -// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor -// it uses separate counts bucket because boltdb Stat call is very slow -func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { - - infoBkt := tx.Bucket([]byte(infoBucketName)) - - info := store.PostInfo{} - if err := b.load(infoBkt, postURL, &info); err != nil { - info = store.PostInfo{} - } - if val == 0 { // get current count, don't update - return info.Count, nil - } - info.Count += val - - return info.Count, b.save(infoBkt, postURL, &info) -} - -func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) { - infoBkt := tx.Bucket([]byte(infoBucketName)) - info := store.PostInfo{} - if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil { - info = store.PostInfo{ - Count: 0, - URL: comment.Locator.URL, - FirstTS: comment.Timestamp, - LastTS: comment.Timestamp, - } - } - info.Count++ - info.LastTS = comment.Timestamp - return info, b.save(infoBkt, comment.Locator.URL, &info) -} - -func (b *BoltDB) db(siteID string) (*bolt.DB, error) { - if res, ok := b.dbs[siteID]; ok { - return res, nil - } - return nil, errors.Errorf("site %q not found", siteID) -} - -// makeRef creates reference combining url and comment id -func (b *BoltDB) makeRef(comment store.Comment) []byte { - return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID)) -} - -// parseRef gets parts of reference -func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) { - elems := strings.Split(string(val), "!!") - if len(elems) != 2 { - return "", "", errors.Errorf("invalid reference value %s", string(val)) - } - return elems[0], elems[1], nil -} diff --git a/backend/app/store/engine_old/bolt_accessor_test.go b/backend/app/store/engine_old/bolt_accessor_test.go deleted file mode 100644 index fa0f6381..00000000 --- a/backend/app/store/engine_old/bolt_accessor_test.go +++ /dev/null @@ -1,406 +0,0 @@ -package engine_old - -import ( - "fmt" - "os" - "testing" - "time" - - bolt "github.com/coreos/bbolt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -var testDb = "test-remark.db" - -func TestBoltDB_CreateAndFind(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, `some text, link`, res[0].Text) - assert.Equal(t, "user1", res[0].User.ID) - t.Log(res[0].ID) - - _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - assert.NotNil(t, err) - assert.Equal(t, "key id-1 already in store", err.Error()) - - _, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time") - assert.EqualError(t, err, `site "radio-t-bad" not found`) - - assert.NoError(t, b.Close()) -} - -func TestBoltDB_CreateReadOnly(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - comment := store.Comment{ - ID: "id-ro", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - err := b.SetReadOnly(comment.Locator, true) - require.Nil(t, err) - - _, err = b.Create(comment) - assert.NotNil(t, err) - assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error()) - - err = b.SetReadOnly(comment.Locator, false) - require.Nil(t, err) - _, err = b.Create(comment) - assert.Nil(t, err) -} - -func TestBoltDB_Get(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) - assert.Nil(t, err) - assert.Equal(t, "some text2", comment.Text) - - comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") - assert.NotNil(t, err) - - _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_Put(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment := res[0] - comment.Text = "abc 123" - comment.Score = 100 - err = b.Put(loc, comment) - assert.Nil(t, err) - - comment, err = b.Get(loc, res[0].ID) - assert.Nil(t, err) - assert.Equal(t, "abc 123", comment.Text) - assert.Equal(t, res[0].ID, comment.ID) - assert.Equal(t, 100, comment.Score) - - err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) - assert.EqualError(t, err, `site "bad" not found`) - - err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) - assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`) -} - -func TestBoltDB_Last(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = b.Last("radio-t", 1, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - _, err = b.Last("bad", 0, time.Time{}) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_LastSince(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local) - res, err := b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - ts = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local) - res, err = b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - ts = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local) - res, err = b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestBoltDB_Count(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 2, c) - - c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c) - - _, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_List(t *testing.T) { - b, teardown := prep(t) // two comments for https://radio-t.com - defer teardown() - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := b.Create(comment) - assert.Nil(t, err) - - ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) } - - res, err := b.List("radio-t", 0, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, - res) - - res, err = b.List("radio-t", -1, -1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = b.List("radio-t", 1, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res) - - res, err = b.List("radio-t", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - _, err = b.List("bad", 1, 1) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_Info(t *testing.T) { - b, teardown := prep(t) // two comments for https://radio-t.com - defer teardown() - - ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) } - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := b.Create(comment) - assert.Nil(t, err) - - r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r) - - r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - - r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r) - - _, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0) - require.NotNil(t, err) - - _, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0) - require.NotNil(t, err) - - err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true) - require.Nil(t, err) - r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - -} - -func TestBoltDB_GetForUser(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = b.User("radio-t", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = b.User("radio-t", "user1", 1, 1) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, `some text, link`, res[0].Text, "second comment") - - _, err = b.User("bad", "user1", 1, 0) - assert.EqualError(t, err, `site "bad" not found`) - - _, err = b.User("radio-t", "userZ", 1, 0) - assert.EqualError(t, err, `no comments for user userZ in store`) -} - -func TestBoltDB_GetForUserPagination(t *testing.T) { - os.Remove(testDb) - b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) - require.Nil(t, err) - - defer func() { - require.NoError(t, b.Close()) - os.Remove(testDb) - }() - - c := store.Comment{ - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - - // write 200 comments - for i := 0; i < 200; i++ { - c.ID = fmt.Sprintf("id-%d", i) - c.Text = fmt.Sprintf("text #%d", i) - c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local) - _, err = b.Create(c) - require.Nil(t, err) - } - - // get all comments - res, err := b.User("radio-t", "user1", 0, 0) - assert.Nil(t, err) - assert.Equal(t, 200, len(res)) - assert.Equal(t, "id-199", res[0].ID) - - // seek 0, 5 comments - res, err = b.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-199", res[0].ID) - assert.Equal(t, "id-195", res[4].ID) - - // seek 10, 3 comments - res, err = b.User("radio-t", "user1", 3, 10) - assert.Nil(t, err) - assert.Equal(t, 3, len(res)) - assert.Equal(t, "id-189", res[0].ID) - assert.Equal(t, "id-187", res[2].ID) - - // seek 195, ask 10 comments - res, err = b.User("radio-t", "user1", 10, 195) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-4", res[0].ID) - assert.Equal(t, "id-0", res[4].ID) - - // seek 255, ask 10 comments - res, err = b.User("radio-t", "user1", 10, 255) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestBoltDB_GetForUserCounter(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - count, err := b.UserCount("radio-t", "user1") - assert.Nil(t, err) - assert.Equal(t, 2, count) - - _, err = b.UserCount("bad", "user1") - assert.EqualError(t, err, `site "bad" not found`) - - _, err = b.UserCount("radio-t", "userZ") - assert.EqualError(t, err, `no comments for user userZ in store`) -} - -func TestBoltDB_Ref(t *testing.T) { - b := BoltDB{} - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - res := b.makeRef(comment) - assert.Equal(t, "https://radio-t.com/2!!12345", string(res)) - - url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345")) - assert.Nil(t, err) - assert.Equal(t, "https://radio-t.com/2", url) - assert.Equal(t, "12345", id) - - _, _, err = b.parseRef([]byte("https://radio-t.com/2")) - assert.NotNil(t, err) -} - -func TestBoltDB_New(t *testing.T) { - _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"}) - assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory") -} - -// makes new boltdb, put two records -func prep(t *testing.T) (b *BoltDB, teardown func()) { - os.Remove(testDb) - - boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) - assert.Nil(t, err) - b = boltStore - - comment := store.Comment{ - ID: "id-1", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - comment = store.Comment{ - ID: "id-2", - Text: "some text2", - Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - teardown = func() { - require.NoError(t, b.Close()) - os.Remove(testDb) - } - return b, teardown -} diff --git a/backend/app/store/engine_old/bolt_admin.go b/backend/app/store/engine_old/bolt_admin.go deleted file mode 100644 index 914346f2..00000000 --- a/backend/app/store/engine_old/bolt_admin.go +++ /dev/null @@ -1,339 +0,0 @@ -package engine_old - -import ( - "encoding/json" - "time" - - bolt "github.com/coreos/bbolt" - log "github.com/go-pkgz/lgr" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -// From last bucket removed for real. -func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - comment := store.Comment{} - if err = b.load(postBkt, commentID, &comment); err != nil { - return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) - } - // set deleted status and clear fields - comment.SetDeleted(mode) - - if err = b.save(postBkt, commentID, comment); err != nil { - return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) - } - - // delete from "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - if err = lastBkt.Delete([]byte(commentID)); err != nil { - return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) - } - - // decrement comments count for post url - if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { - return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) - } - - return nil - }) -} - -// DeleteAll removes all top-level buckets for given siteID -func (b *BoltDB) DeleteAll(siteID string) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // delete all buckets except blocked users - toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName} - - // delete top-level buckets - err = bdb.Update(func(tx *bolt.Tx) error { - for _, bktName := range toDelete { - - if e := tx.DeleteBucket([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) - } - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID) -} - -// DeleteUser removes all comments for given user. Everything will be market as deleted -// and user name and userID will be changed to "deleted". Also removes from last and from user buckets. -func (b *BoltDB) DeleteUser(siteID string, userID string) error { - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // get list of all comments outside of transaction loop - posts, err := b.List(siteID, 0, 0) - if err != nil { - return err - } - - type commentInfo struct { - locator store.Locator - commentID string - } - - // get list of commentID for all user's comment - comments := []commentInfo{} - for _, postInfo := range posts { - err = bdb.View(func(tx *bolt.Tx) error { - postsBkt := tx.Bucket([]byte(postsBucketName)) - postBkt := postsBkt.Bucket([]byte(postInfo.URL)) - err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error { - comment := store.Comment{} - if err = json.Unmarshal(commentVal, &comment); err != nil { - return errors.Wrap(err, "failed to unmarshal") - } - if comment.User.ID == userID { - comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID}) - } - return nil - }) - return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL) - }) - if err != nil { - return err - } - } - - log.Printf("[DEBUG] comments for removal=%d", len(comments)) - - // delete collected comments - for _, ci := range comments { - if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil { - return errors.Wrapf(err, "failed to delete comment %+v", ci) - } - } - - // delete user bucket - err = bdb.Update(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - if usersBkt != nil { - if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { - return errors.Wrapf(err, "failed to delete user bucket for %s", userID) - } - } - return nil - }) - - if err != nil { - return errors.Wrap(err, "can't delete user meta") - } - - if len(comments) == 0 { - return errors.Errorf("unknown user %s", userID) - } - - return err -} - -// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent -// block uses blocksBucketName with key=userID and val=TTL+now -func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - switch status { - case true: - val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year - if ttl > 0 { - val = time.Now().Add(ttl).Format(tsNano) - } - if e := bucket.Put([]byte(userID), []byte(val)); e != nil { - return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) - } - } - return nil - }) -} - -// IsBlocked checks if user blocked -func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - val := bucket.Get([]byte(userID)) - if val == nil { - blocked = false - return nil - } - - until, e := time.Parse(tsNano, string(val)) - if e != nil { - blocked = false - return nil - } - blocked = time.Now().Before(until) - return nil - }) - return blocked -} - -// Blocked get lists of blocked users for given site -// bucket uses userID: -func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - return bucket.ForEach(func(k []byte, v []byte) error { - ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local) - if errParse != nil { - return errors.Wrap(errParse, "can't parse block ts") - } - if time.Now().Before(ts) { - // get user name from comment user section - userName := "" - userComments, errUser := b.User(siteID, string(k), 1, 0) - if errUser == nil && len(userComments) > 0 { - userName = userComments[0].User.Name - } - users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts}) - } - return nil - }) - }) - - return users, err -} - -// SetReadOnly makes post read-only or reset the ro flag -func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error { - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(readonlyBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to set ro for %s", locator.URL) - } - case false: - if e := bucket.Delete([]byte(locator.URL)); e != nil { - return errors.Wrapf(e, "failed to clean ro for %s", locator.URL) - } - } - return nil - }) -} - -// IsReadOnly checks if post in RO mode -func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(readonlyBucketName)) - ro = bucket.Get([]byte(locator.URL)) != nil - return nil - }) - return ro -} - -// SetVerified makes user verified or reset the flag -func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error { - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(verifiedBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to set verified status for %s", userID) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean verified status for %s", userID) - } - } - return nil - }) -} - -// IsVerified checks if user verified -func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(verifiedBucketName)) - verified = bucket.Get([]byte(userID)) != nil - return nil - }) - return verified -} - -// Verified returns list of verified userIDs -func (b *BoltDB) Verified(siteID string) (ids []string, err error) { - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(verifiedBucketName)) - _ = usersBkt.ForEach(func(k, _ []byte) error { - ids = append(ids, string(k)) - return nil - }) - return nil - }) - return ids, err -} diff --git a/backend/app/store/engine_old/bolt_admin_test.go b/backend/app/store/engine_old/bolt_admin_test.go deleted file mode 100644 index a1b4c3c1..00000000 --- a/backend/app/store/engine_old/bolt_admin_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package engine_old - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -func TestBoltAdmin_Delete(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - count, err := b.Count(loc) - require.NoError(t, err) - assert.Equal(t, 2, count, "count=2 initially") - - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.Nil(t, err) - - res, err = b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - - assert.Equal(t, "some text2", res[1].Text) - assert.False(t, res[1].Deleted) - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - - count, err = b.Count(loc) - require.NoError(t, err) - assert.Equal(t, 1, count) - - err = b.Delete(loc, "123456", store.SoftDelete) - assert.NotNil(t, err) - - loc.SiteID = "bad" - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `site "bad" not found`) - - loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"} - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`) -} - -func TestBoltAdmin_DeleteHard(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.Delete(loc, res[0].ID, store.HardDelete) - assert.Nil(t, err) - - res, err = b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) -} - -func TestBoltAdmin_DeleteAll(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.DeleteAll("radio-t") - assert.Nil(t, err) - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - err = b.DeleteAll("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltAdmin_DeleteUser(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - err := b.DeleteUser("radio-t", "user1") - require.NoError(t, err) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "2 comments with deleted info") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User) - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - _, err = b.User("radio-t", "user1", 5, 0) - assert.EqualError(t, err, "no comments for user user1 in store") - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - err = b.DeleteUser("radio-t-bad", "user1") - assert.EqualError(t, err, `site "radio-t-bad" not found`) -} - -func TestBoltAdmin_BlockUser(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0)) - assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") - - assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0)) - assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked") - - assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`) - assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0)) - - assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site") -} - -func TestBoltAdmin_BlockUserWithTTL(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond)) - assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") - time.Sleep(50 * time.Millisecond) - assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically") -} - -func TestBoltAdmin_BlockList(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0)) - assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond)) - assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0)) - - ids, err := b.Blocked("radio-t") - assert.NoError(t, err) - - assert.Equal(t, 2, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - assert.Equal(t, "user2", ids[1].ID) - t.Logf("%+v", ids) - - time.Sleep(50 * time.Millisecond) - ids, err = b.Blocked("radio-t") - assert.NoError(t, err) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - - _, err = b.Blocked("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltAdmin_ReadOnly(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro") - - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true)) - assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro") - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable") - - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false)) - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable") - - assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`) - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false)) - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site") -} - -func TestBoltAdmin_Verified(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified") - - assert.NoError(t, b.SetVerified("radio-t", "u1", true)) - assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified") - - assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified") - assert.NoError(t, b.SetVerified("radio-t", "u1", false)) - assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore") - - assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`) - assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false)) - - assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") - - assert.NoError(t, b.SetVerified("radio-t", "u1", true)) - assert.NoError(t, b.SetVerified("radio-t", "u2", true)) - assert.NoError(t, b.SetVerified("radio-t", "u3", false)) - - ids, err := b.Verified("radio-t") - assert.NoError(t, err) - assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") - - _, err = b.Verified("radio-t-bad") - assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site") -} diff --git a/backend/app/store/engine_old/engine.go b/backend/app/store/engine_old/engine.go deleted file mode 100644 index 2a8dad42..00000000 --- a/backend/app/store/engine_old/engine.go +++ /dev/null @@ -1,87 +0,0 @@ -// Package engine defines interfaces each supported storage should implement. -// Includes default implementation with boltdb -package engine_old - -import ( - "sort" - "strings" - "time" - - "github.com/umputun/remark/backend/app/store" -) - -// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate -//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" - -// Interface defines methods provided by low-level storage engine -type Interface interface { - Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id - Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id - Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only - Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator - Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time - User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time - UserCount(siteID, userID string) (int, error) // comments count by user - Count(locator store.Locator) (int, error) // number of comments for the post - List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts - Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info - Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id - DeleteAll(siteID string) error // delete all data from site - DeleteUser(siteID string, userID string) error // remove all comments from user - SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent) - IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users - SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag - IsReadOnly(locator store.Locator) bool // check if post read-only - SetVerified(siteID string, userID string, status bool) error // set/reset verified flag - IsVerified(siteID string, userID string) bool // check verified status - Verified(siteID string) ([]string, error) // list of verified user ids - Close() error // close/stop engine -} - -const ( - // limits - lastLimit = 1000 - userLimit = 500 -) - -// SortComments is for engines can't sort data internally -func SortComments(comments []store.Comment, sortFld string) []store.Comment { - sort.Slice(comments, func(i, j int) bool { - switch sortFld { - case "+time", "-time", "time", "+active", "-active", "active": - if strings.HasPrefix(sortFld, "-") { - return comments[i].Timestamp.After(comments[j].Timestamp) - } - return comments[i].Timestamp.Before(comments[j].Timestamp) - - case "+score", "-score", "score": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score > comments[j].Score - } - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score < comments[j].Score - - case "+controversy", "-controversy", "controversy": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy > comments[j].Controversy - } - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy < comments[j].Controversy - - default: - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - }) - return comments -} diff --git a/backend/app/store/engine_old/engine_mock.go b/backend/app/store/engine_old/engine_mock.go deleted file mode 100644 index 879402b1..00000000 --- a/backend/app/store/engine_old/engine_mock.go +++ /dev/null @@ -1,408 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package engine_old - -import mock "github.com/stretchr/testify/mock" -import store "github.com/umputun/remark/backend/app/store" -import time "time" - -// MockInterface is an autogenerated mock type for the Interface type -type MockInterface struct { - mock.Mock -} - -// Blocked provides a mock function with given fields: siteID -func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) { - ret := _m.Called(siteID) - - var r0 []store.BlockedUser - if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.BlockedUser) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Close provides a mock function with given fields: -func (_m *MockInterface) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Count provides a mock function with given fields: locator -func (_m *MockInterface) Count(locator store.Locator) (int, error) { - ret := _m.Called(locator) - - var r0 int - if rf, ok := ret.Get(0).(func(store.Locator) int); ok { - r0 = rf(locator) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator) error); ok { - r1 = rf(locator) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Create provides a mock function with given fields: comment -func (_m *MockInterface) Create(comment store.Comment) (string, error) { - ret := _m.Called(comment) - - var r0 string - if rf, ok := ret.Get(0).(func(store.Comment) string); ok { - r0 = rf(comment) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Comment) error); ok { - r1 = rf(comment) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Delete provides a mock function with given fields: locator, commentID, mode -func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - ret := _m.Called(locator, commentID, mode) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok { - r0 = rf(locator, commentID, mode) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteAll provides a mock function with given fields: siteID -func (_m *MockInterface) DeleteAll(siteID string) error { - ret := _m.Called(siteID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(siteID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteUser provides a mock function with given fields: siteID, userID -func (_m *MockInterface) DeleteUser(siteID string, userID string) error { - ret := _m.Called(siteID, userID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Find provides a mock function with given fields: locator, sort -func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) { - ret := _m.Called(locator, sort) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok { - r0 = rf(locator, sort) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, sort) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Get provides a mock function with given fields: locator, commentID -func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { - ret := _m.Called(locator, commentID) - - var r0 store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { - r0 = rf(locator, commentID) - } else { - r0 = ret.Get(0).(store.Comment) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, commentID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Info provides a mock function with given fields: locator, readonlyAge -func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { - ret := _m.Called(locator, readonlyAge) - - var r0 store.PostInfo - if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok { - r0 = rf(locator, readonlyAge) - } else { - r0 = ret.Get(0).(store.PostInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok { - r1 = rf(locator, readonlyAge) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// IsBlocked provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsBlocked(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsReadOnly provides a mock function with given fields: locator -func (_m *MockInterface) IsReadOnly(locator store.Locator) bool { - ret := _m.Called(locator) - - var r0 bool - if rf, ok := ret.Get(0).(func(store.Locator) bool); ok { - r0 = rf(locator) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsVerified provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsVerified(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Last provides a mock function with given fields: siteID, limit, since -func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) { - ret := _m.Called(siteID, limit, since) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok { - r0 = rf(siteID, limit, since) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok { - r1 = rf(siteID, limit, since) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// List provides a mock function with given fields: siteID, limit, skip -func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { - ret := _m.Called(siteID, limit, skip) - - var r0 []store.PostInfo - if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok { - r0 = rf(siteID, limit, skip) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.PostInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int, int) error); ok { - r1 = rf(siteID, limit, skip) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Put provides a mock function with given fields: locator, comment -func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error { - ret := _m.Called(locator, comment) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { - r0 = rf(locator, comment) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetBlock provides a mock function with given fields: siteID, userID, status, ttl -func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - ret := _m.Called(siteID, userID, status, ttl) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok { - r0 = rf(siteID, userID, status, ttl) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetReadOnly provides a mock function with given fields: locator, status -func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error { - ret := _m.Called(locator, status) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok { - r0 = rf(locator, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetVerified provides a mock function with given fields: siteID, userID, status -func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error { - ret := _m.Called(siteID, userID, status) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = rf(siteID, userID, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// User provides a mock function with given fields: siteID, userID, limit, skip -func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) { - ret := _m.Called(siteID, userID, limit, skip) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok { - r0 = rf(siteID, userID, limit, skip) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok { - r1 = rf(siteID, userID, limit, skip) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// UserCount provides a mock function with given fields: siteID, userID -func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) { - ret := _m.Called(siteID, userID) - - var r0 int - if rf, ok := ret.Get(0).(func(string, string) int); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(siteID, userID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Verified provides a mock function with given fields: siteID -func (_m *MockInterface) Verified(siteID string) ([]string, error) { - ret := _m.Called(siteID) - - var r0 []string - if rf, ok := ret.Get(0).(func(string) []string); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/backend/app/store/engine_old/engine_test.go b/backend/app/store/engine_old/engine_test.go deleted file mode 100644 index dc04233b..00000000 --- a/backend/app/store/engine_old/engine_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package engine_old - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/umputun/remark/backend/app/store" -) - -func TestEngine_sortComments(t *testing.T) { - cc := []store.Comment{ - {ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)}, - {ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)}, - {ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)}, - {ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)}, - } - - SortComments(cc, "+time") - assert.Equal(t, "1", cc[0].ID) - assert.Equal(t, "2", cc[1].ID) - assert.Equal(t, "3", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) - - SortComments(cc, "-time") - assert.Equal(t, "4", cc[0].ID) - assert.Equal(t, "3", cc[1].ID) - assert.Equal(t, "2", cc[2].ID) - assert.Equal(t, "1", cc[3].ID) - - SortComments(cc, "score") - assert.Equal(t, "2", cc[0].ID) - assert.Equal(t, "1", cc[1].ID) - assert.Equal(t, "3", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) - - SortComments(cc, "-score") - assert.Equal(t, "3", cc[0].ID) - assert.Equal(t, "4", cc[1].ID) - assert.Equal(t, "1", cc[2].ID) - assert.Equal(t, "2", cc[3].ID) - - SortComments(cc, "controversy") - assert.Equal(t, "1", cc[0].ID) - assert.Equal(t, "4", cc[1].ID) - assert.Equal(t, "2", cc[2].ID) - assert.Equal(t, "3", cc[3].ID) - - SortComments(cc, "-controversy") - assert.Equal(t, "3", cc[0].ID) - assert.Equal(t, "2", cc[1].ID) - assert.Equal(t, "1", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) -} diff --git a/backend/app/store/engine_old/mongo.go b/backend/app/store/engine_old/mongo.go deleted file mode 100644 index 6cd3f311..00000000 --- a/backend/app/store/engine_old/mongo.go +++ /dev/null @@ -1,380 +0,0 @@ -package engine_old - -import ( - "time" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// Mongo implements engine interface -type Mongo struct { - conn *mongo.Connection - postWriter mongo.BufferedWriter -} - -const ( - mongoPosts = "posts" - mongoMetaPosts = "meta_posts" - mongoMetaUsers = "meta_users" -) - -type metaPost struct { - ID string `bson:"_id"` // url - SiteID string `bson:"site"` - ReadOnly bool `bson:"read_only"` -} - -type metaUser struct { - ID string `bson:"_id"` // user_id - SiteID string `bson:"site"` - Verified bool `bson:"verified"` - Blocked bool `bson:"blocked"` - BlockedUntil time.Time `bson:"blocked_until"` -} - -// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off. -// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached. -// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time. -func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) { - writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration) - result := Mongo{conn: conn, postWriter: writer} - err := result.prepare() - return &result, errors.Wrap(err, "failed to prepare mongo") -} - -// Create new comment, write can be buffered and delayed. -func (m *Mongo) Create(comment store.Comment) (commentID string, err error) { - // err = m.postWriter.Write(comment) - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - return coll.Insert(&comment) - }) - return comment.ID, err -} - -// Find returns all comments for post and sorts results -func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { - comments = []store.Comment{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL} - return coll.Find(query).Sort(sortFld).All(&comments) - }) - return comments, err -} - -// Get returns comment for locator.URL and commentID string -func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL} - return coll.Find(query).One(&comment) - }) - return comment, err -} - -// Put updates comment for locator.URL with mutable part of comment -func (m *Mongo) Put(locator store.Locator, comment store.Comment) error { - return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL}, - bson.M{"$set": bson.M{ - "text": comment.Text, - "orig": comment.Orig, - "score": comment.Score, - "votes": comment.Votes, - "pin": comment.Pin, - "deleted": comment.Deleted, - }}) - }) -} - -// Last returns up to max last comments for given siteID -func (m *Mongo) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) { - comments = []store.Comment{} - if max > lastLimit || max == 0 { - max = lastLimit - } - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": siteID, "delete": false} - if !since.IsZero() { - query["time"] = bson.M{"$gt": since} - } - return coll.Find(query).Sort("-time").Limit(max).All(&comments) - }) - return comments, err -} - -// Count returns number of comments for locator -func (m *Mongo) Count(locator store.Locator) (count int, err error) { - - e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false} - count, err = coll.Find(query).Count() - return err - }) - return count, e -} - -// List returns list of all commented posts with counters -func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { - list = []store.PostInfo{} - - if limit <= 0 { - limit = 1000 - } - if skip < 0 { - skip = 0 - } - - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - pipeline := coll.Pipe([]bson.M{ - {"$match": bson.M{"locator.site": siteID}}, - {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}}, - {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1}, - "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}}, - {"$skip": skip}, - {"$limit": limit}, - }) - return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed") - }) - return list, errors.Wrap(err, "can't get list") -} - -// Info returns time range and count for locator -func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) { - list := []store.PostInfo{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - pipeline := coll.Pipe([]bson.M{ - {"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}}, - {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}}, - {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1}, - "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}}, - }) - return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed") - }) - if err != nil { - return info, err - } - if len(list) == 0 { - return info, errors.Errorf("can't load info for %s", locator.URL) - } - info = list[0] - // set read-only from age and manual bucket - info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now()) - if m.IsReadOnly(locator) { - info.ReadOnly = true - } - return info, nil -} - -// User extracts all comments for given site and given userID -func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { - comments = []store.Comment{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": siteID, "user.id": userID} - return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments) - }) - return comments, errors.Wrapf(err, "can't get comments for user %s", userID) -} - -// UserCount returns number of comments for user -func (m *Mongo) UserCount(siteID, userID string) (count int, err error) { - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - var e error - count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count() - return e - }) - return count, errors.Wrapf(err, "can't get comments count for user %s", userID) -} - -// SetReadOnly makes post read-only or reset the ro flag -func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) { - return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}}) - return e - }) -} - -// IsReadOnly checks if post in RO -func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) { - meta := metaPost{} - err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta) - }) - return err == nil && meta.ReadOnly -} - -// SetVerified makes user verified or reset the flag -func (m *Mongo) SetVerified(siteID string, userID string, status bool) error { - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}}) - return e - }) -} - -// IsVerified checks if user verified -func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) { - meta := metaUser{} - err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta) - }) - return err == nil && meta.Verified -} - -// Verified returns list of verified user IDs -func (m *Mongo) Verified(siteID string) (ids []string, err error) { - metas := []metaUser{} - err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas) - }) - if err != nil { - return nil, err - } - for _, meta := range metas { - ids = append(ids, meta.ID) - } - return ids, nil -} - -// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent -// block uses blocksBucketName with key=userID and val=TTL+now -func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - - until := time.Time{} - if status { - until = time.Now().AddDate(100, 0, 0) // permanent is 50year - if ttl > 0 { - until = time.Now().Add(ttl) - } - } - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, - bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}}) - return errors.Wrapf(e, "failed to set block for %s", userID) - }) -} - -// IsBlocked checks if user blocked -func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) { - meta := metaUser{} - err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta) - }) - return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now()) -} - -// Blocked get lists of blocked users for given site -func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - metas := []metaUser{} - err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID, - "blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas) - }) - if err != nil { - return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID) - } - - for _, mu := range metas { - blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil} - if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 { - blockedUser.Name = ucc[0].User.Name - } - users = append(users, blockedUser) - } - return users, nil -} - -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - comment := store.Comment{} - err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment) - if e != nil { - return e - } - comment.SetDeleted(mode) - return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment) - }) - return errors.Wrapf(err, "can't delete %s", commentID) -} - -// DeleteAll removes all info about siteID -func (m *Mongo) DeleteAll(siteID string) error { - err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - _, e := coll.RemoveAll(bson.M{"locator.site": siteID}) - return e - }) - return errors.Wrapf(err, "can't delete site %s", siteID) -} - -// DeleteUser removes all comments for given user. Everything will be market as deleted -// and user name and userID will be changed to "deleted". -func (m *Mongo) DeleteUser(siteID string, userID string) error { - comments := []store.Comment{} - return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments) - if e != nil { - return e - } - for _, c := range comments { - if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil { - return e - } - } - return nil - }) -} - -// Close boltdb store -func (m *Mongo) Close() error { - if m.postWriter != nil { - return m.postWriter.Close() - } - return nil -} - -// prepare collections with all indexes -func (m *Mongo) prepare() error { - errs := new(multierror.Error) - e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts) - }) - if e != nil { - return e - } - - e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts) - }) - if e != nil { - return e - } - - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers) - }) -} - -func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query { - if limit <= 0 { - limit = 1000 - } - if skip < 0 { - skip = 0 - } - return q.Skip(skip).Limit(limit) -} diff --git a/backend/app/store/engine_old/mongo_test.go b/backend/app/store/engine_old/mongo_test.go deleted file mode 100644 index cb16051c..00000000 --- a/backend/app/store/engine_old/mongo_test.go +++ /dev/null @@ -1,604 +0,0 @@ -package engine_old - -import ( - "fmt" - "math/rand" - "testing" - "time" - - "github.com/go-pkgz/mongo" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -func TestMongo_CreateAndFind(t *testing.T) { - var m Interface - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - require.Equal(t, 2, len(res)) - assert.Equal(t, `some text, link`, res[0].Text) - assert.Equal(t, "user1", res[0].User.ID) - t.Log(res[0].ID) - - _, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - assert.NotNil(t, err, "reject dup") - - id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}}) - assert.Nil(t, err) - assert.Equal(t, "id-3", id) - res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time") - assert.Nil(t, err) - require.Equal(t, 1, len(res)) - - assert.NoError(t, m.Close()) -} - -func TestMongo_Get(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) - assert.Nil(t, err) - assert.Equal(t, "some text2", comment.Text) - - comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") - assert.NotNil(t, err, "not found") -} - -func TestMongo_Put(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment := res[0] - comment.Text = "abc 123" - comment.Score = 100 - err = m.Put(loc, comment) - assert.Nil(t, err) - - comment, err = m.Get(loc, res[0].ID) - assert.Nil(t, err) - assert.Equal(t, "abc 123", comment.Text) - assert.Equal(t, res[0].ID, comment.ID) - assert.Equal(t, 100, comment.Score) - - err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) - assert.EqualError(t, err, `not found`) - - err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) - assert.EqualError(t, err, `not found`) -} - -func TestMongo_Last(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 1, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) -} - -func TestMongo_Count(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 2, c) - - c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c) -} - -func TestMongo_List(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := m.Create(comment) - assert.Nil(t, err) - - ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) } - - res, err := m.List("radio-t", 0, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, - res) - - res, err = m.List("radio-t", -1, -1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = m.List("radio-t", 1, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res) - - res, err = m.List("radio-t", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = m.List("bad", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{}, res) -} - -func TestMongo_Info(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) } - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := m.Create(comment) - assert.Nil(t, err) - - r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r) - - r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - - r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r) - - _, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0) - require.NotNil(t, err) - - _, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0) - require.NotNil(t, err) - - err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true) - require.Nil(t, err) - r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) -} - -func TestMongo_ReadOnly(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro") - - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true)) - assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro") - - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable") - - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false)) - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable") - - assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"") - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false)) - - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site") -} - -func TestMongo_Verified(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified") - - assert.NoError(t, m.SetVerified("radio-t", "u1", true)) - assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified") - - assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified") - assert.NoError(t, m.SetVerified("radio-t", "u1", false)) - assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore") - - assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`) - assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false)) - - assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") - - assert.NoError(t, m.SetVerified("radio-t", "u1", true)) - assert.NoError(t, m.SetVerified("radio-t", "u2", true)) - assert.NoError(t, m.SetVerified("radio-t", "u3", false)) - - ids, err := m.Verified("radio-t") - assert.NoError(t, err) - assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") - - ids, err = m.Verified("radio-t-bad") - assert.NoError(t, err) - assert.Equal(t, 0, len(ids)) -} - -func TestMongo_GetForUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = m.User("radio-t", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = m.User("radio-t", "user1", 1, 1) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, `some text, link`, res[0].Text, "second comment") - - res, err = m.User("bad", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestMongo_GetForUserPagination(t *testing.T) { - m, skip := prepMongo(t, false) - if skip { - return - } - c := store.Comment{ - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - - // write 200 comments - for i := 0; i < 200; i++ { - c.ID = fmt.Sprintf("id-%d", i) - c.Text = fmt.Sprintf("text #%d", i) - c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local) - _, err := m.Create(c) - require.Nil(t, err, c.ID) - } - - // get all comments - res, err := m.User("radio-t", "user1", 0, 0) - assert.Nil(t, err) - assert.Equal(t, 200, len(res)) - assert.Equal(t, "id-199", res[0].ID) - - // seek 0, 5 comments - res, err = m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-199", res[0].ID) - assert.Equal(t, "id-195", res[4].ID) - - // seek 10, 3 comments - res, err = m.User("radio-t", "user1", 3, 10) - assert.Nil(t, err) - assert.Equal(t, 3, len(res)) - assert.Equal(t, "id-189", res[0].ID) - assert.Equal(t, "id-187", res[2].ID) - - // seek 195, ask 10 comments - res, err = m.User("radio-t", "user1", 10, 195) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-4", res[0].ID) - assert.Equal(t, "id-0", res[4].ID) - - // seek 255, ask 10 comments - res, err = m.User("radio-t", "user1", 10, 255) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestMongo_BlockUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked") - - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0)) - assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked") - - assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked") - - assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0)) - assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked") - - assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`) - assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0)) - - assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site") -} - -func TestMongo_BlockUserWithTTL(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked") - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond)) - assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked") - time.Sleep(500 * time.Millisecond) - assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically") -} - -func TestMongo_GetForUserCounter(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - count, err := m.UserCount("radio-t", "user1") - assert.Nil(t, err) - assert.Equal(t, 2, count) - - count, err = m.UserCount("bad", "user1") - assert.Nil(t, err) - assert.Equal(t, 0, count) -} - -func TestMongo_BlockList(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0)) - assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond)) - assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0)) - - ids, err := m.Blocked("radio-t") - assert.NoError(t, err) - - assert.Equal(t, 2, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - assert.Equal(t, "user2", ids[1].ID) - t.Logf("%+v", ids) - - time.Sleep(500 * time.Millisecond) - ids, err = m.Blocked("radio-t") - assert.NoError(t, err) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - - ids, err = m.Blocked("bad") - assert.NoError(t, err) - assert.Equal(t, 0, len(ids)) -} - -func TestMongo_Delete(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.Nil(t, err) - - res, err = m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - - assert.Equal(t, "some text2", res[1].Text) - assert.False(t, res[1].Deleted) - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - - err = m.Delete(loc, "123456", store.SoftDelete) - assert.NotNil(t, err) - - loc.SiteID = "bad" - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `can't delete id-1: not found`) - - loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"} - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `can't delete id-1: not found`) -} - -func TestMongo_DeleteHard(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.Delete(loc, res[0].ID, store.HardDelete) - assert.Nil(t, err) - - res, err = m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) -} - -func TestMongo_DeleteAll(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.DeleteAll("radio-t") - assert.Nil(t, err) - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") -} - -func TestMongo_DeleteUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - err := m.DeleteUser("radio-t", "user1") - require.NoError(t, err) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "2 comments with deleted info") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User) - - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - cc, err := m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err, "no comments for user user1 in store") - assert.Equal(t, 0, len(cc), "no comments for user user1 in store") - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") -} - -func TestMongo_Parallel(t *testing.T) { - var m Interface - var skip bool - m, skip = prepMongoBuffered(t) // buffered engine, no comments - if skip { - return - } - go func() { - for i := 0; i < 100; i++ { - _, err := m.Create(store.Comment{ - ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - require.Nil(t, err) - time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond) - } - }() - - for { - time.Sleep(10 * time.Millisecond) - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - if len(res) == 100 { - break - } - } -} - -func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) { - conn, err := mongo.MakeTestConnection(t) - if err != nil { - return nil, true - } - mongo.RemoveTestCollection(t, conn) - - m, err := NewMongo(conn, 1, 0*time.Microsecond) - require.Nil(t, err) - - mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers) - comment := store.Comment{ - ID: "id-1", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - if writeRecs { - _, err = m.Create(comment) - assert.Nil(t, err) - } - - comment = store.Comment{ - ID: "id-2", - Text: "some text2", - Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - if writeRecs { - _, err = m.Create(comment) - assert.Nil(t, err) - } - - return m, false -} - -func prepMongoBuffered(t *testing.T) (*Mongo, bool) { - conn, err := mongo.MakeTestConnection(t) - if err != nil { - return nil, true - } - mongo.RemoveTestCollection(t, conn) - - m, err := NewMongo(conn, 10, 10*time.Millisecond) - mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers) - - require.Nil(t, err) - return m, false -} diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go index e200e03f..dd033926 100644 --- a/backend/app/store/remote/client.go +++ b/backend/app/store/remote/client.go @@ -27,7 +27,7 @@ func (r *Client) Call(method string, args ...interface{}) (*Response, error) { var err error switch { - case args == nil || len(args) == 0: + case len(args) == 0: b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)}) if err != nil { return nil, errors.Wrapf(err, "marshaling failed for %s", method) diff --git a/backend/go.mod b/backend/go.mod index d00e249f..8fccd3d5 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -11,14 +11,13 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/didip/tollbooth v4.0.0+incompatible github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d - github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/cors v1.0.0 github.com/go-chi/render v1.0.1 github.com/go-pkgz/auth v0.5.2 github.com/go-pkgz/lcw v0.3.1 github.com/go-pkgz/lgr v0.6.2 - github.com/go-pkgz/mongo v1.1.2 + github.com/go-pkgz/mongo v1.1.2 // indirect github.com/go-pkgz/repeater v1.1.2 github.com/go-pkgz/rest v1.4.1 github.com/go-pkgz/syncs v1.1.1