mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #56 from umputun/feature/delateuser

Browse Source 
feature/delateuser - ability to remove all user data
This commit is contained in:
Umputun
2018-05-26 19:09:28 -05:00
committed by GitHub
parent 29659a8f33 5c739af4a0
commit 90569b9c62
11 changed files with 526 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+25 -24
View File
@@ -30,29 +30,29 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
#### Parameters
| Command line | Environment | Default | Multi | Description |
| ----------------- | -------------------- | ---------------------- | ---------------- | --------------------------------------- |
| --url | REMARK_URL | `https://remark42.com` | no | url to remark server |
| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory |
| --site | SITE | `remark` | yes | site name(s) |
| --admin | ADMIN | | yes | admin names (list of user ids) |
| --backup | BACKUP_PATH | `/tmp` | no | backups location |
| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep |
| --max-cache-items | MAX_CACHE_ITEMS | `1000` | no | max number of cached items, 0-unlimited |
| --max-cache-value | MAX_CACHE_VALUE | `65536` | no | max size of cached value, o-unlimited |
| --secret | SECRET | | no | secret key, required |
| --max-comment | MAX_COMMENT_SIZE | 2048 | no | comment's size limit |
| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID |
| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret |
| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID |
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret |
| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID |
| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret |
| --low-score | LOW_SCORE | `-5` | no | Low score threshold |
| --critical-score | CRITICAL_SCORE | `-10` | no | Critical score threshold |
| --img-proxy | IMG_PROXY | `false` | no | Enable http->https proxy for images |
| --dbg | DEBUG | `false` | no | debug mode |
| --dev-passwd | DEV_PASSWD | | no | password for `dev` user |
| Command line | Environment | Default | Multi | Description |
| ----------------- | -------------------- | ---------------------- | ----- | --------------------------------------- |
| --url | REMARK_URL | `https://remark42.com` | no | url to remark server |
| --bolt | BOLTDB_PATH | `/tmp` | no | path to data directory |
| --site | SITE | `remark` | yes | site name(s) |
| --admin | ADMIN | | yes | admin names (list of user ids) |
| --backup | BACKUP_PATH | `/tmp` | no | backups location |
| --max-back | MAX_BACKUP_FILES | `10` | no | max backup files to keep |
| --max-cache-items | MAX_CACHE_ITEMS | `1000` | no | max number of cached items, 0-unlimited |
| --max-cache-value | MAX_CACHE_VALUE | `65536` | no | max size of cached value, o-unlimited |
| --secret | SECRET | | no | secret key, required |
| --max-comment | MAX_COMMENT_SIZE | 2048 | no | comment's size limit |
| --google-cid | REMARK_GOOGLE_CID | | no | Google OAuth client ID |
| --google-csec | REMARK_GOOGLE_CSEC | | no | Google OAuth client secret |
| --facebook-cid | REMARK_FACEBOOK_CID | | no | Facebook OAuth client ID |
| --facebook-csec | REMARK_FACEBOOK_CSEC | | no | Facebook OAuth client secret |
| --github-cid | REMARK_GITHUB_CID | | no | Github OAuth client ID |
| --github-csec | REMARK_GITHUB_CSEC | | no | Github OAuth client secret |
| --low-score | LOW_SCORE | `-5` | no | Low score threshold |
| --critical-score | CRITICAL_SCORE | `-10` | no | Critical score threshold |
| --img-proxy | IMG_PROXY | `false` | no | Enable http->https proxy for images |
| --dbg | DEBUG | `false` | no | debug mode |
| --dev-passwd | DEV_PASSWD | | no | password for `dev` user |
**user has to provide secret key, can be any long and hard-to-guess string.**
@@ -357,12 +357,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
* `POST /api/v1/admin/import?site=side-id` - import comments from the backup.
* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
* `DELETE /api/v1/admin/user/{userid}?site=site-id&block=1` - delete all user's comments.
_all admin calls require auth and admin privilege_
## Technical details
* Data stored in [boltdb](https://github.com/boltdb/bolt) (embedded key/value database) files under `BOLTDB_PATH`
* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `BOLTDB_PATH`
* Each site stored in a separate boltbd file.
* In order to migrate/move remark42 to another host boltbd files should be transferred.
* Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
app/rest/api/admin.go
+19 -4
View File
@@ -31,8 +31,8 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
router.Use(middlewares...)
router.Delete("/comment/{id}", a.deleteCommentCtrl)
router.Put("/user/{userid}", a.setBlockCtrl)
router.Delete("/user/{userid}", a.deleteUserCtrl)
router.Get("/export", a.exportCtrl)
router.Put("/pin/{id}", a.setPinCtrl)
router.Get("/blocked", a.blockedUsersCtrl)
return router
@@ -45,7 +45,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[INFO] delete comment %s", id)
err := a.dataService.Delete(locator, id)
err := a.dataService.Delete(locator, id, store.SoftDelete)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
return
@@ -55,6 +55,22 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"id": id, "locator": locator})
}
// DELETE /user/{userid}?site=side-id
func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
userID := chi.URLParam(r, "userid")
siteID := r.URL.Query().Get("site")
log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
if err := a.dataService.DeleteUser(siteID, userID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
return
}
a.cache.Flush(siteID, userID)
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
}
// PUT /user/{userid}?site=side-id&block=1 - block or unblock user
func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
userID := chi.URLParam(r, "userid")
@@ -118,7 +134,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
}
func (a *admin) checkBlocked(siteID string, user store.User) bool {
@@ -138,7 +153,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []
// process blocked users
if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) {
if !isAdmin { // reset comment to deleted for non-admins
c.SetDeleted()
c.SetDeleted(store.SoftDelete)
}
c.User.Blocked = true
c.Deleted = true
app/rest/api/admin_test.go
+54
View File
@@ -46,6 +46,60 @@ func TestAdmin_Delete(t *testing.T) {
assert.True(t, cr.Deleted)
}
func TestAdmin_DeleteUser(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
c2 := store.Comment{Text: "test test #2", Orig: "o test test #2", User: store.User{ID: "id2", Name: "name"}, ParentID: "p1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
c3 := store.Comment{Text: "test test #3", Orig: "o test test #3", User: store.User{ID: "id2", Name: "name"}, ParentID: "",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
// write comments directly to store to keep user id
id1, err := srv.DataService.Create(c1)
assert.NoError(t, err)
_, err = srv.DataService.Create(c2)
assert.NoError(t, err)
_, err = srv.DataService.Create(c3)
assert.NoError(t, err)
client := http.Client{}
req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
assert.Nil(t, err)
withBasicAuth(req, "dev", "password")
resp, err := client.Do(req)
assert.Nil(t, err)
assert.Equal(t, 200, resp.StatusCode)
// all 3 comments here, but for id2 they deleted
res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
assert.Equal(t, 200, code)
comments := []store.Comment{}
err = json.Unmarshal([]byte(res), &comments)
assert.Nil(t, err)
assert.Equal(t, 3, len(comments), "should have 3 comment")
// id1 comment untouched
assert.Equal(t, id1, comments[0].ID)
assert.Equal(t, "o test test #1", comments[0].Orig)
assert.False(t, comments[0].Deleted)
t.Logf("%+v", comments[0].User)
// id2 comments fully deleted
assert.Equal(t, "", comments[1].Text)
assert.Equal(t, "", comments[1].Orig)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
assert.True(t, comments[1].Deleted)
assert.Equal(t, "", comments[2].Text)
assert.Equal(t, "", comments[2].Orig)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
assert.True(t, comments[2].Deleted)
}
func TestAdmin_Pin(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
app/rest/auth/provider_test.go
+2 -2
View File
@@ -135,7 +135,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
switch {
case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
state := r.URL.Query().Get("state")
w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", loginPort, state))
w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s",
loginPort, state))
w.WriteHeader(302)
case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"):
res := `{
@@ -150,7 +151,6 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
w.WriteHeader(200)
w.Write([]byte(res))
case strings.HasPrefix(r.URL.Path, "/user"):
res := fmt.Sprintf(`{
"id": "%s",
"name":"blah",
app/store/comment.go
+18 -2
View File
@@ -49,6 +49,15 @@ type BlockedUser struct {
Timestamp time.Time `json:"time"`
}
// DeleteMode defines how much comment info will be erased
type DeleteMode int
// DeleteMode enum
const (
SoftDelete DeleteMode = 0
HardDelete DeleteMode = 1
)
// PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
// autogen fields and reset everything users not supposed to provide
func (c *Comment) PrepareUntrusted() {
@@ -61,8 +70,8 @@ func (c *Comment) PrepareUntrusted() {
c.Deleted = false
}
// SetDeleted clears comment info, reset to deleted state
func (c *Comment) SetDeleted() {
// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well
func (c *Comment) SetDeleted(mode DeleteMode) {
c.Text = ""
c.Orig = ""
c.Score = 0
@@ -70,6 +79,13 @@ func (c *Comment) SetDeleted() {
c.Edit = nil
c.Deleted = true
c.Pin = false
if mode == HardDelete {
c.User.Name = "deleted"
c.User.ID = "deleted"
c.User.Picture = ""
c.User.IP = ""
}
}
// Sanitize clean dangerous html/js from the comment
app/store/comment_test.go
+29 -2
View File
@@ -62,7 +62,7 @@ func TestComment_PrepareUntrusted(t *testing.T) {
func TestComment_SetDeleted(t *testing.T) {
comment := Comment{
Text: `blah`,
User: User{ID: "username"},
User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
@@ -73,7 +73,7 @@ func TestComment_SetDeleted(t *testing.T) {
Pin: true,
}
comment.SetDeleted()
comment.SetDeleted(SoftDelete)
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
@@ -82,4 +82,31 @@ func TestComment_SetDeleted(t *testing.T) {
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User)
}
func TestComment_SetDeletedHard(t *testing.T) {
comment := Comment{
Text: `blah`,
User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
Score: 10,
Deleted: false,
Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
Votes: map[string]bool{"uu": true},
Pin: true,
}
comment.SetDeleted(HardDelete)
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
assert.Equal(t, map[string]bool{}, comment.Votes)
assert.Equal(t, 0, comment.Score)
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
}
app/store/engine/bolt.go → app/store/engine/bolt_accessor.go
+1 -144
View File
@@ -6,7 +6,6 @@ import (
"log"
"strconv"
"strings"
"time"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
@@ -134,77 +133,6 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
return comment.ID, err
}
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator store.Locator, commentID string) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
comment, err := b.load(postBkt, []byte(commentID))
if err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted()
if err := b.save(postBkt, []byte(commentID), comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
// delete from "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
if err := lastBkt.Delete([]byte(commentID)); err != nil {
return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
// DeleteAll removes all top-level buckets for given siteID
func (b *BoltDB) DeleteAll(siteID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
}
}
return nil
})
return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
}
// Find returns all comments for post and sorts results
func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
comments = []store.Comment{}
@@ -296,77 +224,6 @@ func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
return count, err
}
// SetBlock blocks/unblocks user for given site
func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
switch status {
case true:
if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
}
}
return nil
})
}
// IsBlocked checks if user blocked
func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
blocked = bucket.Get([]byte(userID)) != nil
return nil
})
return blocked
}
// Blocked get lists of blocked users for given site
// bucket uses userID:
func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
if e != nil {
return errors.Wrap(e, "can't parse block ts")
}
// get user name from comment user section
userName := ""
if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 {
userName = userComments[0].User.Name
}
users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
return nil
})
})
return users, err
}
// List returns list of all commented posts with counters
// uses count bucket to get number of comments
func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
@@ -550,7 +407,7 @@ func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, comment store.Comment) (err
func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment store.Comment, err error) {
commentVal := bkt.Get(key)
if commentVal == nil {
return comment, errors.Errorf("no comment for %s", key)
return comment, errors.Errorf("no comments for %s", key)
}
if err = json.Unmarshal(commentVal, &comment); err != nil {
app/store/engine/bolt_test.go → app/store/engine/bolt_accessor_test.go
-94
View File
@@ -32,62 +32,6 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
assert.EqualError(t, err, `site "radio-t-bad" not found`)
}
func TestBoltDB_Delete(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
comments, err := b.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
err = b.Delete(loc, "123456")
assert.NotNil(t, err)
loc.SiteID = "bad"
err = b.Delete(loc, res[0].ID)
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_DeleteAll(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.DeleteAll("radio-t")
assert.Nil(t, err)
comments, err := b.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
err = b.DeleteAll("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_Get(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
@@ -165,44 +109,6 @@ func TestBoltDB_Count(t *testing.T) {
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_BlockUser(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", true))
assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", false))
assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`)
assert.NoError(t, b.SetBlock("radio-t", "userX", false))
}
func TestBoltDB_BlockList(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
assert.NoError(t, b.SetBlock("radio-t", "user1", true))
assert.NoError(t, b.SetBlock("radio-t", "user2", true))
assert.NoError(t, b.SetBlock("radio-t", "user3", false))
ids, err := b.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 2, len(ids))
assert.Equal(t, "user1", ids[0].ID)
assert.Equal(t, "user2", ids[1].ID)
t.Logf("%+v", ids)
_, err = b.Blocked("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_List(t *testing.T) {
defer os.Remove(testDb)
b := prep(t) // two comments for https://radio-t.com
app/store/engine/bolt_admin.go
+219
View File
@@ -0,0 +1,219 @@
package engine
import (
"encoding/json"
"log"
"time"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/umputun/remark/app/store"
)
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
comment, err := b.load(postBkt, []byte(commentID))
if err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted(mode)
if err := b.save(postBkt, []byte(commentID), comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
// delete from "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
if err := lastBkt.Delete([]byte(commentID)); err != nil {
return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
// DeleteAll removes all top-level buckets for given siteID
func (b *BoltDB) DeleteAll(siteID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
}
}
return nil
})
return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
}
// DeleteUser removes all comments for given user. Everyting will be market as deleted
// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
func (b *BoltDB) DeleteUser(siteID string, userID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// get list of all comments outside of transaction loop
posts, err := b.List(siteID, 0, 0)
if err != nil {
return err
}
type commentInfo struct {
locator store.Locator
commentID string
}
// get list of commentID for all user's comment
comments := []commentInfo{}
for _, postInfo := range posts {
err = bdb.View(func(tx *bolt.Tx) error {
postsBkt := tx.Bucket([]byte(postsBucketName))
postBkt := postsBkt.Bucket([]byte(postInfo.URL))
err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
comment := store.Comment{}
if err = json.Unmarshal(commentVal, &comment); err != nil {
return errors.Wrap(err, "failed to unmarshal")
}
if comment.User.ID == userID {
comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
}
return nil
})
return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
})
if err != nil {
return err
}
}
log.Printf("[DEBUG] comments for removal=%d", len(comments))
// delete collected comments
for _, ci := range comments {
if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
return errors.Wrapf(err, "failed to delete comment %+v", ci)
}
}
// delete user bucket
err = bdb.Update(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
if usersBkt != nil {
if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
return errors.Wrapf(err, "failed to delete user bucker for %s", userID)
}
}
return nil
})
return nil
}
// SetBlock blocks/unblocks user for given site
func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
switch status {
case true:
if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
}
}
return nil
})
}
// IsBlocked checks if user blocked
func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
blocked = bucket.Get([]byte(userID)) != nil
return nil
})
return blocked
}
// Blocked get lists of blocked users for given site
// bucket uses userID:
func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
if e != nil {
return errors.Wrap(e, "can't parse block ts")
}
// get user name from comment user section
userName := ""
if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 {
userName = userComments[0].User.Name
}
users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
return nil
})
})
return users, err
}
app/store/engine/bolt_admin_test.go
+153
View File
@@ -0,0 +1,153 @@
package engine
import (
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/app/store"
)
func TestBoltDB_Delete(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
comments, err := b.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
err = b.Delete(loc, "123456", store.SoftDelete)
assert.NotNil(t, err)
loc.SiteID = "bad"
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_DeleteHard(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID, store.HardDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
}
func TestBoltDB_DeleteAll(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.DeleteAll("radio-t")
assert.Nil(t, err)
comments, err := b.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
err = b.DeleteAll("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_DeleteUser(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
err := b.DeleteUser("radio-t", "user1")
require.NoError(t, err)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
_, _, err = b.User("radio-t", "user1", 5)
assert.EqualError(t, err, "no comments for user user1 in store")
comments, err := b.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
}
func TestBoltDB_BlockUser(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", true))
assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
assert.NoError(t, b.SetBlock("radio-t", "user1", false))
assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`)
assert.NoError(t, b.SetBlock("radio-t", "userX", false))
}
func TestBoltDB_BlockList(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
assert.NoError(t, b.SetBlock("radio-t", "user1", true))
assert.NoError(t, b.SetBlock("radio-t", "user2", true))
assert.NoError(t, b.SetBlock("radio-t", "user3", false))
ids, err := b.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 2, len(ids))
assert.Equal(t, "user1", ids[0].ID)
assert.Equal(t, "user2", ids[1].ID)
t.Logf("%+v", ids)
_, err = b.Blocked("bad")
assert.EqualError(t, err, `site "bad" not found`)
}
app/store/engine/engine.go
+6 -5
View File
@@ -31,11 +31,12 @@ type Accessor interface {
// Admin defines all store ops avail for admin only
type Admin interface {
Delete(locator store.Locator, commentID string) error // delete comment by id
DeleteAll(siteID string) error // delete all data from site
SetBlock(siteID string, userID string, status bool) error // block or unblock user
IsBlocked(siteID string, userID string) bool // check if user blocked
Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
DeleteAll(siteID string) error // delete all data from site
DeleteUser(siteID string, userID string) error // remove all comments from user
SetBlock(siteID string, userID string, status bool) error // block or unblock user
IsBlocked(siteID string, userID string) bool // check if user blocked
Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
}
// sortComments is for engines can't sort data internally