diff --git a/README.md b/README.md index b6a05b22..354f0e45 100644 --- a/README.md +++ b/README.md @@ -30,29 +30,29 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi #### Parameters -| Command line | Environment | Default | Multi | Description | -| ----------------- | -------------------- | ---------------------- | ---------------- | --------------------------------------- | -| --url | REMARK_URL | `https://remark42.com` | no | url to remark server | -| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory | -| --site | SITE | `remark` | yes | site name(s) | -| --admin | ADMIN | | yes | admin names (list of user ids) | -| --backup | BACKUP_PATH | `/tmp` | no | backups location | -| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep | -| --max-cache-items | MAX_CACHE_ITEMS | `1000` | no | max number of cached items, 0-unlimited | -| --max-cache-value | MAX_CACHE_VALUE | `65536` | no | max size of cached value, o-unlimited | -| --secret | SECRET | | no | secret key, required | -| --max-comment | MAX_COMMENT_SIZE | 2048 | no | comment's size limit | -| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID | -| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret | -| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID | -| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret | -| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID | -| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret | -| --low-score | LOW_SCORE | `-5` | no | Low score threshold | -| --critical-score | CRITICAL_SCORE | `-10` | no | Critical score threshold | -| --img-proxy | IMG_PROXY | `false` | no | Enable http->https proxy for images | -| --dbg | DEBUG | `false` | no | debug mode | -| --dev-passwd | DEV_PASSWD | | no | password for `dev` user | +| Command line | Environment | Default | Multi | Description | +| ----------------- | -------------------- | ---------------------- | ----- | --------------------------------------- | +| --url | REMARK_URL | `https://remark42.com` | no | url to remark server | +| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory | +| --site | SITE | `remark` | yes | site name(s) | +| --admin | ADMIN | | yes | admin names (list of user ids) | +| --backup | BACKUP_PATH | `/tmp` | no | backups location | +| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep | +| --max-cache-items | MAX_CACHE_ITEMS | `1000` | no | max number of cached items, 0-unlimited | +| --max-cache-value | MAX_CACHE_VALUE | `65536` | no | max size of cached value, o-unlimited | +| --secret | SECRET | | no | secret key, required | +| --max-comment | MAX_COMMENT_SIZE | 2048 | no | comment's size limit | +| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID | +| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret | +| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID | +| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret | +| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID | +| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret | +| --low-score | LOW_SCORE | `-5` | no | Low score threshold | +| --critical-score | CRITICAL_SCORE | `-10` | no | Critical score threshold | +| --img-proxy | IMG_PROXY | `false` | no | Enable http->https proxy for images | +| --dbg | DEBUG | `false` | no | debug mode | +| --dev-passwd | DEV_PASSWD | | no | password for `dev` user | **user has to provide secret key, can be any long and hard-to-guess string.** @@ -357,12 +357,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i * `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file. * `POST /api/v1/admin/import?site=side-id` - import comments from the backup. * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment. +* `DELETE /api/v1/admin/user/{userid}?site=site-id&block=1` - delete all user's comments. _all admin calls require auth and admin privilege_ ## Technical details -* Data stored in [boltdb](https://github.com/boltdb/bolt) (embedded key/value database) files under `BOLTDB_PATH` +* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `BOLTDB_PATH` * Each site stored in a separate boltbd file. * In order to migrate/move remark42 to another host boltbd files should be transferred. * Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`. diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 5aeaa680..d72e372e 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -31,8 +31,8 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route router.Use(middlewares...) router.Delete("/comment/{id}", a.deleteCommentCtrl) router.Put("/user/{userid}", a.setBlockCtrl) + router.Delete("/user/{userid}", a.deleteUserCtrl) router.Get("/export", a.exportCtrl) - router.Put("/pin/{id}", a.setPinCtrl) router.Get("/blocked", a.blockedUsersCtrl) return router @@ -45,7 +45,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} log.Printf("[INFO] delete comment %s", id) - err := a.dataService.Delete(locator, id) + err := a.dataService.Delete(locator, id, store.SoftDelete) if err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment") return @@ -55,6 +55,22 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"id": id, "locator": locator}) } +// DELETE /user/{userid}?site=side-id +func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) { + + userID := chi.URLParam(r, "userid") + siteID := r.URL.Query().Get("site") + log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID) + + if err := a.dataService.DeleteUser(siteID, userID); err != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user") + return + } + a.cache.Flush(siteID, userID) + render.Status(r, http.StatusOK) + render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID}) +} + // PUT /user/{userid}?site=side-id&block=1 - block or unblock user func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) { userID := chi.URLParam(r, "userid") @@ -118,7 +134,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } - } func (a *admin) checkBlocked(siteID string, user store.User) bool { @@ -138,7 +153,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res [] // process blocked users if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) { if !isAdmin { // reset comment to deleted for non-admins - c.SetDeleted() + c.SetDeleted(store.SoftDelete) } c.User.Blocked = true c.Deleted = true diff --git a/app/rest/api/admin_test.go b/app/rest/api/admin_test.go index c8be5690..f968ec6f 100644 --- a/app/rest/api/admin_test.go +++ b/app/rest/api/admin_test.go @@ -46,6 +46,60 @@ func TestAdmin_Delete(t *testing.T) { assert.True(t, cr.Deleted) } +func TestAdmin_DeleteUser(t *testing.T) { + srv, ts := prep(t) + assert.NotNil(t, srv) + defer cleanup(ts) + + c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"}, + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} + c2 := store.Comment{Text: "test test #2", Orig: "o test test #2", User: store.User{ID: "id2", Name: "name"}, ParentID: "p1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} + c3 := store.Comment{Text: "test test #3", Orig: "o test test #3", User: store.User{ID: "id2", Name: "name"}, ParentID: "", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}} + + // write comments directly to store to keep user id + id1, err := srv.DataService.Create(c1) + assert.NoError(t, err) + _, err = srv.DataService.Create(c2) + assert.NoError(t, err) + _, err = srv.DataService.Create(c3) + assert.NoError(t, err) + + client := http.Client{} + req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil) + assert.Nil(t, err) + withBasicAuth(req, "dev", "password") + resp, err := client.Do(req) + assert.Nil(t, err) + assert.Equal(t, 200, resp.StatusCode) + + // all 3 comments here, but for id2 they deleted + res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time") + assert.Equal(t, 200, code) + comments := []store.Comment{} + err = json.Unmarshal([]byte(res), &comments) + assert.Nil(t, err) + assert.Equal(t, 3, len(comments), "should have 3 comment") + + // id1 comment untouched + assert.Equal(t, id1, comments[0].ID) + assert.Equal(t, "o test test #1", comments[0].Orig) + assert.False(t, comments[0].Deleted) + t.Logf("%+v", comments[0].User) + + // id2 comments fully deleted + assert.Equal(t, "", comments[1].Text) + assert.Equal(t, "", comments[1].Orig) + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User) + assert.True(t, comments[1].Deleted) + + assert.Equal(t, "", comments[2].Text) + assert.Equal(t, "", comments[2].Orig) + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User) + assert.True(t, comments[2].Deleted) +} + func TestAdmin_Pin(t *testing.T) { srv, ts := prep(t) assert.NotNil(t, srv) diff --git a/app/rest/auth/provider_test.go b/app/rest/auth/provider_test.go index c8e8b7e2..72017f17 100644 --- a/app/rest/auth/provider_test.go +++ b/app/rest/auth/provider_test.go @@ -135,7 +135,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se switch { case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"): state := r.URL.Query().Get("state") - w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", loginPort, state)) + w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", + loginPort, state)) w.WriteHeader(302) case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"): res := `{ @@ -150,7 +151,6 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se w.WriteHeader(200) w.Write([]byte(res)) case strings.HasPrefix(r.URL.Path, "/user"): - res := fmt.Sprintf(`{ "id": "%s", "name":"blah", diff --git a/app/store/comment.go b/app/store/comment.go index 9f61aea8..92c83c9b 100644 --- a/app/store/comment.go +++ b/app/store/comment.go @@ -49,6 +49,15 @@ type BlockedUser struct { Timestamp time.Time `json:"time"` } +// DeleteMode defines how much comment info will be erased +type DeleteMode int + +// DeleteMode enum +const ( + SoftDelete DeleteMode = 0 + HardDelete DeleteMode = 1 +) + // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all // autogen fields and reset everything users not supposed to provide func (c *Comment) PrepareUntrusted() { @@ -61,8 +70,8 @@ func (c *Comment) PrepareUntrusted() { c.Deleted = false } -// SetDeleted clears comment info, reset to deleted state -func (c *Comment) SetDeleted() { +// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well +func (c *Comment) SetDeleted(mode DeleteMode) { c.Text = "" c.Orig = "" c.Score = 0 @@ -70,6 +79,13 @@ func (c *Comment) SetDeleted() { c.Edit = nil c.Deleted = true c.Pin = false + + if mode == HardDelete { + c.User.Name = "deleted" + c.User.ID = "deleted" + c.User.Picture = "" + c.User.IP = "" + } } // Sanitize clean dangerous html/js from the comment diff --git a/app/store/comment_test.go b/app/store/comment_test.go index aec8e79a..5723f7c0 100644 --- a/app/store/comment_test.go +++ b/app/store/comment_test.go @@ -62,7 +62,7 @@ func TestComment_PrepareUntrusted(t *testing.T) { func TestComment_SetDeleted(t *testing.T) { comment := Comment{ Text: `blah`, - User: User{ID: "username"}, + User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"}, ParentID: "p123", ID: "123", Locator: Locator{SiteID: "site", URL: "url"}, @@ -73,7 +73,7 @@ func TestComment_SetDeleted(t *testing.T) { Pin: true, } - comment.SetDeleted() + comment.SetDeleted(SoftDelete) assert.Equal(t, "", comment.Text) assert.Equal(t, "", comment.Orig) @@ -82,4 +82,31 @@ func TestComment_SetDeleted(t *testing.T) { assert.True(t, comment.Deleted) assert.Nil(t, comment.Edit) assert.False(t, comment.Pin) + assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User) +} + +func TestComment_SetDeletedHard(t *testing.T) { + comment := Comment{ + Text: `blah`, + User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"}, + ParentID: "p123", + ID: "123", + Locator: Locator{SiteID: "site", URL: "url"}, + Score: 10, + Deleted: false, + Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local), + Votes: map[string]bool{"uu": true}, + Pin: true, + } + + comment.SetDeleted(HardDelete) + + assert.Equal(t, "", comment.Text) + assert.Equal(t, "", comment.Orig) + assert.Equal(t, map[string]bool{}, comment.Votes) + assert.Equal(t, 0, comment.Score) + assert.True(t, comment.Deleted) + assert.Nil(t, comment.Edit) + assert.False(t, comment.Pin) + assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User) } diff --git a/app/store/engine/bolt.go b/app/store/engine/bolt_accessor.go similarity index 75% rename from app/store/engine/bolt.go rename to app/store/engine/bolt_accessor.go index 1b14b199..a7c4ea0d 100644 --- a/app/store/engine/bolt.go +++ b/app/store/engine/bolt_accessor.go @@ -6,7 +6,6 @@ import ( "log" "strconv" "strings" - "time" "github.com/coreos/bbolt" "github.com/pkg/errors" @@ -134,77 +133,6 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { return comment.ID, err } -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -// From last bucket removed for real. -func (b *BoltDB) Delete(locator store.Locator, commentID string) error { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - comment, err := b.load(postBkt, []byte(commentID)) - if err != nil { - return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) - } - // set deleted status and clear fields - comment.SetDeleted() - - if err := b.save(postBkt, []byte(commentID), comment); err != nil { - return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) - } - - // delete from "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - if err := lastBkt.Delete([]byte(commentID)); err != nil { - return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) - } - - // decrement comments count for post url - if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { - return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) - } - - return nil - }) -} - -// DeleteAll removes all top-level buckets for given siteID -func (b *BoltDB) DeleteAll(siteID string) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // delete all buckets except blocked users - toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName} - - // delete top-level buckets - err = bdb.Update(func(tx *bolt.Tx) error { - for _, bktName := range toDelete { - - if e := tx.DeleteBucket([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) - } - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID) -} - // Find returns all comments for post and sorts results func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { comments = []store.Comment{} @@ -296,77 +224,6 @@ func (b *BoltDB) Count(locator store.Locator) (count int, err error) { return count, err } -// SetBlock blocks/unblocks user for given site -func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) - } - } - return nil - }) -} - -// IsBlocked checks if user blocked -func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - blocked = bucket.Get([]byte(userID)) != nil - return nil - }) - return blocked -} - -// Blocked get lists of blocked users for given site -// bucket uses userID: -func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - return bucket.ForEach(func(k []byte, v []byte) error { - ts, e := time.ParseInLocation(tsNano, string(v), time.Local) - if e != nil { - return errors.Wrap(e, "can't parse block ts") - } - - // get user name from comment user section - userName := "" - if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 { - userName = userComments[0].User.Name - } - - users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) - return nil - }) - }) - - return users, err -} - // List returns list of all commented posts with counters // uses count bucket to get number of comments func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { @@ -550,7 +407,7 @@ func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, comment store.Comment) (err func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment store.Comment, err error) { commentVal := bkt.Get(key) if commentVal == nil { - return comment, errors.Errorf("no comment for %s", key) + return comment, errors.Errorf("no comments for %s", key) } if err = json.Unmarshal(commentVal, &comment); err != nil { diff --git a/app/store/engine/bolt_test.go b/app/store/engine/bolt_accessor_test.go similarity index 71% rename from app/store/engine/bolt_test.go rename to app/store/engine/bolt_accessor_test.go index 39195462..c1d38b2e 100644 --- a/app/store/engine/bolt_test.go +++ b/app/store/engine/bolt_accessor_test.go @@ -32,62 +32,6 @@ func TestBoltDB_CreateAndFind(t *testing.T) { assert.EqualError(t, err, `site "radio-t-bad" not found`) } -func TestBoltDB_Delete(t *testing.T) { - defer os.Remove(testDb) - b := prep(t) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.Delete(loc, res[0].ID) - assert.Nil(t, err) - - res, err = b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, "some text2", res[1].Text) - assert.False(t, res[1].Deleted) - - comments, err := b.Last("radio-t", 10) - assert.Nil(t, err) - assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - - err = b.Delete(loc, "123456") - assert.NotNil(t, err) - - loc.SiteID = "bad" - err = b.Delete(loc, res[0].ID) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_DeleteAll(t *testing.T) { - defer os.Remove(testDb) - b := prep(t) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.DeleteAll("radio-t") - assert.Nil(t, err) - - comments, err := b.Last("radio-t", 10) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - err = b.DeleteAll("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - func TestBoltDB_Get(t *testing.T) { defer os.Remove(testDb) b := prep(t) @@ -165,44 +109,6 @@ func TestBoltDB_Count(t *testing.T) { assert.EqualError(t, err, `site "bad" not found`) } -func TestBoltDB_BlockUser(t *testing.T) { - defer os.Remove(testDb) - b := prep(t) - - assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", true)) - assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") - - assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", false)) - assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked") - - assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`) - assert.NoError(t, b.SetBlock("radio-t", "userX", false)) -} - -func TestBoltDB_BlockList(t *testing.T) { - defer os.Remove(testDb) - b := prep(t) - - assert.NoError(t, b.SetBlock("radio-t", "user1", true)) - assert.NoError(t, b.SetBlock("radio-t", "user2", true)) - assert.NoError(t, b.SetBlock("radio-t", "user3", false)) - - ids, err := b.Blocked("radio-t") - assert.NoError(t, err) - - assert.Equal(t, 2, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - assert.Equal(t, "user2", ids[1].ID) - t.Logf("%+v", ids) - - _, err = b.Blocked("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - func TestBoltDB_List(t *testing.T) { defer os.Remove(testDb) b := prep(t) // two comments for https://radio-t.com diff --git a/app/store/engine/bolt_admin.go b/app/store/engine/bolt_admin.go new file mode 100644 index 00000000..211708cc --- /dev/null +++ b/app/store/engine/bolt_admin.go @@ -0,0 +1,219 @@ +package engine + +import ( + "encoding/json" + "log" + "time" + + "github.com/coreos/bbolt" + "github.com/pkg/errors" + + "github.com/umputun/remark/app/store" +) + +// Delete removes comment, by locator from the store. +// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. +// From last bucket removed for real. +func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { + + bdb, err := b.db(locator.SiteID) + if err != nil { + return err + } + + return bdb.Update(func(tx *bolt.Tx) error { + + postBkt, e := b.getPostBucket(tx, locator.URL) + if e != nil { + return e + } + + comment, err := b.load(postBkt, []byte(commentID)) + if err != nil { + return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) + } + // set deleted status and clear fields + comment.SetDeleted(mode) + + if err := b.save(postBkt, []byte(commentID), comment); err != nil { + return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) + } + + // delete from "last" bucket + lastBkt := tx.Bucket([]byte(lastBucketName)) + if err := lastBkt.Delete([]byte(commentID)); err != nil { + return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) + } + + // decrement comments count for post url + if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { + return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) + } + + return nil + }) +} + +// DeleteAll removes all top-level buckets for given siteID +func (b *BoltDB) DeleteAll(siteID string) error { + + bdb, err := b.db(siteID) + if err != nil { + return err + } + + // delete all buckets except blocked users + toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName} + + // delete top-level buckets + err = bdb.Update(func(tx *bolt.Tx) error { + for _, bktName := range toDelete { + + if e := tx.DeleteBucket([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to create top level bucket %s", bktName) + } + } + return nil + }) + + return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID) +} + +// DeleteUser removes all comments for given user. Everyting will be market as deleted +// and user name and userID will be changed to "deleted". Also removes from last and from user buckets. +func (b *BoltDB) DeleteUser(siteID string, userID string) error { + bdb, err := b.db(siteID) + if err != nil { + return err + } + + // get list of all comments outside of transaction loop + posts, err := b.List(siteID, 0, 0) + if err != nil { + return err + } + + type commentInfo struct { + locator store.Locator + commentID string + } + + // get list of commentID for all user's comment + comments := []commentInfo{} + for _, postInfo := range posts { + err = bdb.View(func(tx *bolt.Tx) error { + postsBkt := tx.Bucket([]byte(postsBucketName)) + postBkt := postsBkt.Bucket([]byte(postInfo.URL)) + err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error { + comment := store.Comment{} + if err = json.Unmarshal(commentVal, &comment); err != nil { + return errors.Wrap(err, "failed to unmarshal") + } + if comment.User.ID == userID { + comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID}) + } + return nil + }) + return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL) + }) + if err != nil { + return err + } + } + + log.Printf("[DEBUG] comments for removal=%d", len(comments)) + + // delete collected comments + for _, ci := range comments { + if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil { + return errors.Wrapf(err, "failed to delete comment %+v", ci) + } + } + + // delete user bucket + err = bdb.Update(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + if usersBkt != nil { + if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { + return errors.Wrapf(err, "failed to delete user bucker for %s", userID) + } + } + return nil + }) + + return nil +} + +// SetBlock blocks/unblocks user for given site +func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error { + + bdb, err := b.db(siteID) + if err != nil { + return err + } + + return bdb.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + switch status { + case true: + if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { + return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) + } + case false: + if e := bucket.Delete([]byte(userID)); e != nil { + return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) + } + } + return nil + }) +} + +// IsBlocked checks if user blocked +func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { + + bdb, err := b.db(siteID) + if err != nil { + return false + } + + _ = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + blocked = bucket.Get([]byte(userID)) != nil + return nil + }) + return blocked +} + +// Blocked get lists of blocked users for given site +// bucket uses userID: +func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { + users = []store.BlockedUser{} + bdb, err := b.db(siteID) + if err != nil { + return nil, err + } + + err = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + return bucket.ForEach(func(k []byte, v []byte) error { + ts, e := time.ParseInLocation(tsNano, string(v), time.Local) + if e != nil { + return errors.Wrap(e, "can't parse block ts") + } + + // get user name from comment user section + userName := "" + if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 { + userName = userComments[0].User.Name + } + + users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) + return nil + }) + }) + + return users, err +} diff --git a/app/store/engine/bolt_admin_test.go b/app/store/engine/bolt_admin_test.go new file mode 100644 index 00000000..dac4955b --- /dev/null +++ b/app/store/engine/bolt_admin_test.go @@ -0,0 +1,153 @@ +package engine + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/app/store" +) + +func TestBoltDB_Delete(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + err = b.Delete(loc, res[0].ID, store.SoftDelete) + assert.Nil(t, err) + + res, err = b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "", res[0].Text) + assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) + + assert.Equal(t, "some text2", res[1].Text) + assert.False(t, res[1].Deleted) + + comments, err := b.Last("radio-t", 10) + assert.Nil(t, err) + assert.Equal(t, 1, len(comments), "1 in last, 1 removed") + + err = b.Delete(loc, "123456", store.SoftDelete) + assert.NotNil(t, err) + + loc.SiteID = "bad" + err = b.Delete(loc, res[0].ID, store.SoftDelete) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltDB_DeleteHard(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + err = b.Delete(loc, res[0].ID, store.HardDelete) + assert.Nil(t, err) + + res, err = b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "", res[0].Text) + assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) +} + +func TestBoltDB_DeleteAll(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + err = b.DeleteAll("radio-t") + assert.Nil(t, err) + + comments, err := b.Last("radio-t", 10) + assert.Nil(t, err) + assert.Equal(t, 0, len(comments), "nothing left") + + c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + assert.Nil(t, err) + assert.Equal(t, 0, c, "0 count") + + err = b.DeleteAll("bad") + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltDB_DeleteUser(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + err := b.DeleteUser("radio-t", "user1") + require.NoError(t, err) + + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "2 comments with deleted info") + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User) + + c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + assert.Nil(t, err) + assert.Equal(t, 0, c, "0 count") + + _, _, err = b.User("radio-t", "user1", 5) + assert.EqualError(t, err, "no comments for user user1 in store") + + comments, err := b.Last("radio-t", 10) + assert.Nil(t, err) + assert.Equal(t, 0, len(comments), "nothing left") +} + +func TestBoltDB_BlockUser(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") + + assert.NoError(t, b.SetBlock("radio-t", "user1", true)) + assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") + + assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked") + + assert.NoError(t, b.SetBlock("radio-t", "user1", false)) + assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked") + + assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`) + assert.NoError(t, b.SetBlock("radio-t", "userX", false)) +} + +func TestBoltDB_BlockList(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + assert.NoError(t, b.SetBlock("radio-t", "user1", true)) + assert.NoError(t, b.SetBlock("radio-t", "user2", true)) + assert.NoError(t, b.SetBlock("radio-t", "user3", false)) + + ids, err := b.Blocked("radio-t") + assert.NoError(t, err) + + assert.Equal(t, 2, len(ids)) + assert.Equal(t, "user1", ids[0].ID) + assert.Equal(t, "user2", ids[1].ID) + t.Logf("%+v", ids) + + _, err = b.Blocked("bad") + assert.EqualError(t, err, `site "bad" not found`) +} diff --git a/app/store/engine/engine.go b/app/store/engine/engine.go index 4a520268..10caa24f 100644 --- a/app/store/engine/engine.go +++ b/app/store/engine/engine.go @@ -31,11 +31,12 @@ type Accessor interface { // Admin defines all store ops avail for admin only type Admin interface { - Delete(locator store.Locator, commentID string) error // delete comment by id - DeleteAll(siteID string) error // delete all data from site - SetBlock(siteID string, userID string, status bool) error // block or unblock user - IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users + Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id + DeleteAll(siteID string) error // delete all data from site + DeleteUser(siteID string, userID string) error // remove all comments from user + SetBlock(siteID string, userID string, status bool) error // block or unblock user + IsBlocked(siteID string, userID string) bool // check if user blocked + Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users } // sortComments is for engines can't sort data internally