mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: start to implement hard delete to all user's comments #47

Browse Source
This commit is contained in:
Umputun
2018-05-26 02:03:03 -05:00
parent a6f588e452
commit b3dc433ec9
8 changed files with 299 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/rest/api/admin.go
+2 -3
View File
@@ -45,7 +45,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[INFO] delete comment %s", id)
err := a.dataService.Delete(locator, id)
err := a.dataService.Delete(locator, id, store.SoftDelete)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
return
@@ -118,7 +118,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
}
func (a *admin) checkBlocked(siteID string, user store.User) bool {
@@ -138,7 +137,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []
// process blocked users
if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) {
if !isAdmin { // reset comment to deleted for non-admins
c.SetDeleted()
c.SetDeleted(store.SoftDelete)
}
c.User.Blocked = true
c.Deleted = true
app/rest/auth/provider_test.go
+2 -2
View File
@@ -135,7 +135,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
switch {
case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
state := r.URL.Query().Get("state")
w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", loginPort, state))
w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s",
loginPort, state))
w.WriteHeader(302)
case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"):
res := `{
@@ -150,7 +151,6 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
w.WriteHeader(200)
w.Write([]byte(res))
case strings.HasPrefix(r.URL.Path, "/user"):
res := fmt.Sprintf(`{
"id": "%s",
"name":"blah",
app/store/comment.go
+18 -2
View File
@@ -49,6 +49,15 @@ type BlockedUser struct {
Timestamp time.Time `json:"time"`
}
// DeleteMode defines how much comment info will be erased
type DeleteMode int
// DeleteMode enum
const (
SoftDelete DeleteMode = 0
HardDelete DeleteMode = 1
)
// PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
// autogen fields and reset everything users not supposed to provide
func (c *Comment) PrepareUntrusted() {
@@ -61,8 +70,8 @@ func (c *Comment) PrepareUntrusted() {
c.Deleted = false
}
// SetDeleted clears comment info, reset to deleted state
func (c *Comment) SetDeleted() {
// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well
func (c *Comment) SetDeleted(mode DeleteMode) {
c.Text = ""
c.Orig = ""
c.Score = 0
@@ -70,6 +79,13 @@ func (c *Comment) SetDeleted() {
c.Edit = nil
c.Deleted = true
c.Pin = false
if mode == HardDelete {
c.User.Name = "deleted"
c.User.ID = "deleted"
c.User.Picture = ""
c.User.IP = ""
}
}
// Sanitize clean dangerous html/js from the comment
app/store/comment_test.go
+29 -2
View File
@@ -62,7 +62,7 @@ func TestComment_PrepareUntrusted(t *testing.T) {
func TestComment_SetDeleted(t *testing.T) {
comment := Comment{
Text: `blah`,
User: User{ID: "username"},
User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
@@ -73,7 +73,7 @@ func TestComment_SetDeleted(t *testing.T) {
Pin: true,
}
comment.SetDeleted()
comment.SetDeleted(SoftDelete)
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
@@ -82,4 +82,31 @@ func TestComment_SetDeleted(t *testing.T) {
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User)
}
func TestComment_SetDeletedHard(t *testing.T) {
comment := Comment{
Text: `blah`,
User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
Score: 10,
Deleted: false,
Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
Votes: map[string]bool{"uu": true},
Pin: true,
}
comment.SetDeleted(HardDelete)
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
assert.Equal(t, map[string]bool{}, comment.Votes)
assert.Equal(t, 0, comment.Score)
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
}
app/store/engine/bolt.go → app/store/engine/bolt_accessor.go
-143
View File
@@ -6,7 +6,6 @@ import (
"log"
"strconv"
"strings"
"time"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
@@ -134,77 +133,6 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
return comment.ID, err
}
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator store.Locator, commentID string) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
comment, err := b.load(postBkt, []byte(commentID))
if err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted()
if err := b.save(postBkt, []byte(commentID), comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
// delete from "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
if err := lastBkt.Delete([]byte(commentID)); err != nil {
return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
// DeleteAll removes all top-level buckets for given siteID
func (b *BoltDB) DeleteAll(siteID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
}
}
return nil
})
return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
}
// Find returns all comments for post and sorts results
func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
comments = []store.Comment{}
@@ -296,77 +224,6 @@ func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
return count, err
}
// SetBlock blocks/unblocks user for given site
func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
switch status {
case true:
if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
}
}
return nil
})
}
// IsBlocked checks if user blocked
func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
blocked = bucket.Get([]byte(userID)) != nil
return nil
})
return blocked
}
// Blocked get lists of blocked users for given site
// bucket uses userID:
func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
if e != nil {
return errors.Wrap(e, "can't parse block ts")
}
// get user name from comment user section
userName := ""
if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 {
userName = userComments[0].User.Name
}
users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
return nil
})
})
return users, err
}
// List returns list of all commented posts with counters
// uses count bucket to get number of comments
func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
app/store/engine/bolt_admin.go
+217
View File
@@ -0,0 +1,217 @@
package engine
import (
"time"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/umputun/remark/app/store"
)
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
bdb, err := b.db(locator.SiteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
postBkt, e := b.getPostBucket(tx, locator.URL)
if e != nil {
return e
}
comment, err := b.load(postBkt, []byte(commentID))
if err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.SetDeleted(mode)
if err := b.save(postBkt, []byte(commentID), comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
}
// delete from "last" bucket
lastBkt := tx.Bucket([]byte(lastBucketName))
if err := lastBkt.Delete([]byte(commentID)); err != nil {
return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
// DeleteAll removes all top-level buckets for given siteID
func (b *BoltDB) DeleteAll(siteID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// delete all buckets except blocked users
toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName}
// delete top-level buckets
err = bdb.Update(func(tx *bolt.Tx) error {
for _, bktName := range toDelete {
if e := tx.DeleteBucket([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
}
}
return nil
})
return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
}
// DeleteUser removes all comments for given user. Everyting will be market as deleted
// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
func (b *BoltDB) DeleteUser(siteID string, userID string) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
// get list of all comments outside of transaction loop
posts, err := b.List(siteID, 0, 0)
if err != nil {
return err
}
type commentInfo struct {
locator store.Locator
commentID string
}
// get list of commentID for all user's comment
comments := []commentInfo{}
err = bdb.View(func(tx *bolt.Tx) error {
postsBkt := tx.Bucket([]byte(postsBucketName))
for _, postInfo := range posts {
postBkt := postsBkt.Bucket([]byte(postInfo.URL))
err = postsBkt.ForEach(func(k []byte, v []byte) error {
comment, err := b.load(postBkt, k)
if err != nil {
return errors.Wrapf(err, "can't load key %s from bucket %s", k, postInfo.URL)
}
if comment.User.ID == userID {
comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
}
return nil
})
if err != nil {
return errors.Wrapf(err, "failed to delete comments from %s", postInfo.URL)
}
}
return nil
})
if err != nil {
return errors.Wrapf(err, "failed to collect list of all comments for deletion")
}
// delete collected comments
for _, ci := range comments {
b.Delete(ci.locator, ci.commentID, store.HardDelete)
}
// delete user bucket
err = bdb.Update(func(tx *bolt.Tx) error {
usersBkt := tx.Bucket([]byte(userBucketName))
if usersBkt != nil {
if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
return errors.Wrapf(err, "failed to delete user bucker for %s", userID)
}
}
return nil
})
return nil
}
// SetBlock blocks/unblocks user for given site
func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
bdb, err := b.db(siteID)
if err != nil {
return err
}
return bdb.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
switch status {
case true:
if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
}
case false:
if e := bucket.Delete([]byte(userID)); e != nil {
return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
}
}
return nil
})
}
// IsBlocked checks if user blocked
func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
bdb, err := b.db(siteID)
if err != nil {
return false
}
_ = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
blocked = bucket.Get([]byte(userID)) != nil
return nil
})
return blocked
}
// Blocked get lists of blocked users for given site
// bucket uses userID:
func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
bdb, err := b.db(siteID)
if err != nil {
return nil, err
}
err = bdb.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(blocksBucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
if e != nil {
return errors.Wrap(e, "can't parse block ts")
}
// get user name from comment user section
userName := ""
if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 {
userName = userComments[0].User.Name
}
users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
return nil
})
})
return users, err
}
app/store/engine/bolt_test.go
+25 -3
View File
@@ -41,7 +41,7 @@ func TestBoltDB_Delete(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID)
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
@@ -49,6 +49,8 @@ func TestBoltDB_Delete(t *testing.T) {
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
@@ -56,14 +58,34 @@ func TestBoltDB_Delete(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
err = b.Delete(loc, "123456")
err = b.Delete(loc, "123456", store.SoftDelete)
assert.NotNil(t, err)
loc.SiteID = "bad"
err = b.Delete(loc, res[0].ID)
err = b.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `site "bad" not found`)
}
func TestBoltDB_DeleteHard(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = b.Delete(loc, res[0].ID, store.HardDelete)
assert.Nil(t, err)
res, err = b.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
}
func TestBoltDB_DeleteAll(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
app/store/engine/engine.go
+6 -5
View File
@@ -31,11 +31,12 @@ type Accessor interface {
// Admin defines all store ops avail for admin only
type Admin interface {
Delete(locator store.Locator, commentID string) error // delete comment by id
DeleteAll(siteID string) error // delete all data from site
SetBlock(siteID string, userID string, status bool) error // block or unblock user
IsBlocked(siteID string, userID string) bool // check if user blocked
Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
DeleteAll(siteID string) error // delete all data from site
DeleteUser(siteID string, userID string) error // remove all comments from user
SetBlock(siteID string, userID string, status bool) error // block or unblock user
IsBlocked(siteID string, userID string) bool // check if user blocked
Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
}
// sortComments is for engines can't sort data internally