From b3dc433ec940d2435414af2fa0d0ff732415fb60 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 26 May 2018 02:03:03 -0500 Subject: [PATCH] WIP: start to implement hard delete to all user's comments #47 --- app/rest/api/admin.go | 5 +- app/rest/auth/provider_test.go | 4 +- app/store/comment.go | 20 +- app/store/comment_test.go | 31 ++- .../engine/{bolt.go => bolt_accessor.go} | 143 ------------ app/store/engine/bolt_admin.go | 217 ++++++++++++++++++ app/store/engine/bolt_test.go | 28 ++- app/store/engine/engine.go | 11 +- 8 files changed, 299 insertions(+), 160 deletions(-) rename app/store/engine/{bolt.go => bolt_accessor.go} (76%) create mode 100644 app/store/engine/bolt_admin.go diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 5aeaa680..bfd20a0c 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -45,7 +45,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) { locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")} log.Printf("[INFO] delete comment %s", id) - err := a.dataService.Delete(locator, id) + err := a.dataService.Delete(locator, id, store.SoftDelete) if err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment") return @@ -118,7 +118,6 @@ func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed") return } - } func (a *admin) checkBlocked(siteID string, user store.User) bool { @@ -138,7 +137,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res [] // process blocked users if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) { if !isAdmin { // reset comment to deleted for non-admins - c.SetDeleted() + c.SetDeleted(store.SoftDelete) } c.User.Blocked = true c.Deleted = true diff --git a/app/rest/auth/provider_test.go b/app/rest/auth/provider_test.go index c8e8b7e2..72017f17 100644 --- a/app/rest/auth/provider_test.go +++ b/app/rest/auth/provider_test.go @@ -135,7 +135,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se switch { case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"): state := r.URL.Query().Get("state") - w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", loginPort, state)) + w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s", + loginPort, state)) w.WriteHeader(302) case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"): res := `{ @@ -150,7 +151,6 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se w.WriteHeader(200) w.Write([]byte(res)) case strings.HasPrefix(r.URL.Path, "/user"): - res := fmt.Sprintf(`{ "id": "%s", "name":"blah", diff --git a/app/store/comment.go b/app/store/comment.go index 9f61aea8..92c83c9b 100644 --- a/app/store/comment.go +++ b/app/store/comment.go @@ -49,6 +49,15 @@ type BlockedUser struct { Timestamp time.Time `json:"time"` } +// DeleteMode defines how much comment info will be erased +type DeleteMode int + +// DeleteMode enum +const ( + SoftDelete DeleteMode = 0 + HardDelete DeleteMode = 1 +) + // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all // autogen fields and reset everything users not supposed to provide func (c *Comment) PrepareUntrusted() { @@ -61,8 +70,8 @@ func (c *Comment) PrepareUntrusted() { c.Deleted = false } -// SetDeleted clears comment info, reset to deleted state -func (c *Comment) SetDeleted() { +// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well +func (c *Comment) SetDeleted(mode DeleteMode) { c.Text = "" c.Orig = "" c.Score = 0 @@ -70,6 +79,13 @@ func (c *Comment) SetDeleted() { c.Edit = nil c.Deleted = true c.Pin = false + + if mode == HardDelete { + c.User.Name = "deleted" + c.User.ID = "deleted" + c.User.Picture = "" + c.User.IP = "" + } } // Sanitize clean dangerous html/js from the comment diff --git a/app/store/comment_test.go b/app/store/comment_test.go index aec8e79a..5723f7c0 100644 --- a/app/store/comment_test.go +++ b/app/store/comment_test.go @@ -62,7 +62,7 @@ func TestComment_PrepareUntrusted(t *testing.T) { func TestComment_SetDeleted(t *testing.T) { comment := Comment{ Text: `blah`, - User: User{ID: "username"}, + User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"}, ParentID: "p123", ID: "123", Locator: Locator{SiteID: "site", URL: "url"}, @@ -73,7 +73,7 @@ func TestComment_SetDeleted(t *testing.T) { Pin: true, } - comment.SetDeleted() + comment.SetDeleted(SoftDelete) assert.Equal(t, "", comment.Text) assert.Equal(t, "", comment.Orig) @@ -82,4 +82,31 @@ func TestComment_SetDeleted(t *testing.T) { assert.True(t, comment.Deleted) assert.Nil(t, comment.Edit) assert.False(t, comment.Pin) + assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User) +} + +func TestComment_SetDeletedHard(t *testing.T) { + comment := Comment{ + Text: `blah`, + User: User{ID: "userid", Name: "username", IP: "123", Picture: "pic"}, + ParentID: "p123", + ID: "123", + Locator: Locator{SiteID: "site", URL: "url"}, + Score: 10, + Deleted: false, + Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local), + Votes: map[string]bool{"uu": true}, + Pin: true, + } + + comment.SetDeleted(HardDelete) + + assert.Equal(t, "", comment.Text) + assert.Equal(t, "", comment.Orig) + assert.Equal(t, map[string]bool{}, comment.Votes) + assert.Equal(t, 0, comment.Score) + assert.True(t, comment.Deleted) + assert.Nil(t, comment.Edit) + assert.False(t, comment.Pin) + assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User) } diff --git a/app/store/engine/bolt.go b/app/store/engine/bolt_accessor.go similarity index 76% rename from app/store/engine/bolt.go rename to app/store/engine/bolt_accessor.go index 1b14b199..eebdb787 100644 --- a/app/store/engine/bolt.go +++ b/app/store/engine/bolt_accessor.go @@ -6,7 +6,6 @@ import ( "log" "strconv" "strings" - "time" "github.com/coreos/bbolt" "github.com/pkg/errors" @@ -134,77 +133,6 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { return comment.ID, err } -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -// From last bucket removed for real. -func (b *BoltDB) Delete(locator store.Locator, commentID string) error { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - comment, err := b.load(postBkt, []byte(commentID)) - if err != nil { - return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) - } - // set deleted status and clear fields - comment.SetDeleted() - - if err := b.save(postBkt, []byte(commentID), comment); err != nil { - return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) - } - - // delete from "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - if err := lastBkt.Delete([]byte(commentID)); err != nil { - return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) - } - - // decrement comments count for post url - if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { - return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) - } - - return nil - }) -} - -// DeleteAll removes all top-level buckets for given siteID -func (b *BoltDB) DeleteAll(siteID string) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // delete all buckets except blocked users - toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName} - - // delete top-level buckets - err = bdb.Update(func(tx *bolt.Tx) error { - for _, bktName := range toDelete { - - if e := tx.DeleteBucket([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) - } - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID) -} - // Find returns all comments for post and sorts results func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { comments = []store.Comment{} @@ -296,77 +224,6 @@ func (b *BoltDB) Count(locator store.Locator) (count int, err error) { return count, err } -// SetBlock blocks/unblocks user for given site -func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) - } - } - return nil - }) -} - -// IsBlocked checks if user blocked -func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - blocked = bucket.Get([]byte(userID)) != nil - return nil - }) - return blocked -} - -// Blocked get lists of blocked users for given site -// bucket uses userID: -func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - return bucket.ForEach(func(k []byte, v []byte) error { - ts, e := time.ParseInLocation(tsNano, string(v), time.Local) - if e != nil { - return errors.Wrap(e, "can't parse block ts") - } - - // get user name from comment user section - userName := "" - if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 { - userName = userComments[0].User.Name - } - - users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) - return nil - }) - }) - - return users, err -} - // List returns list of all commented posts with counters // uses count bucket to get number of comments func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { diff --git a/app/store/engine/bolt_admin.go b/app/store/engine/bolt_admin.go new file mode 100644 index 00000000..12043f46 --- /dev/null +++ b/app/store/engine/bolt_admin.go @@ -0,0 +1,217 @@ +package engine + +import ( + "time" + + "github.com/coreos/bbolt" + "github.com/pkg/errors" + + "github.com/umputun/remark/app/store" +) + +// Delete removes comment, by locator from the store. +// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. +// From last bucket removed for real. +func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { + + bdb, err := b.db(locator.SiteID) + if err != nil { + return err + } + + return bdb.Update(func(tx *bolt.Tx) error { + + postBkt, e := b.getPostBucket(tx, locator.URL) + if e != nil { + return e + } + + comment, err := b.load(postBkt, []byte(commentID)) + if err != nil { + return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) + } + // set deleted status and clear fields + comment.SetDeleted(mode) + + if err := b.save(postBkt, []byte(commentID), comment); err != nil { + return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) + } + + // delete from "last" bucket + lastBkt := tx.Bucket([]byte(lastBucketName)) + if err := lastBkt.Delete([]byte(commentID)); err != nil { + return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) + } + + // decrement comments count for post url + if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { + return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) + } + + return nil + }) +} + +// DeleteAll removes all top-level buckets for given siteID +func (b *BoltDB) DeleteAll(siteID string) error { + + bdb, err := b.db(siteID) + if err != nil { + return err + } + + // delete all buckets except blocked users + toDelete := []string{postsBucketName, lastBucketName, userBucketName, countsBucketName} + + // delete top-level buckets + err = bdb.Update(func(tx *bolt.Tx) error { + for _, bktName := range toDelete { + + if e := tx.DeleteBucket([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { + return errors.Wrapf(err, "failed to create top level bucket %s", bktName) + } + } + return nil + }) + + return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID) +} + +// DeleteUser removes all comments for given user. Everyting will be market as deleted +// and user name and userID will be changed to "deleted". Also removes from last and from user buckets. +func (b *BoltDB) DeleteUser(siteID string, userID string) error { + bdb, err := b.db(siteID) + if err != nil { + return err + } + + // get list of all comments outside of transaction loop + posts, err := b.List(siteID, 0, 0) + if err != nil { + return err + } + + type commentInfo struct { + locator store.Locator + commentID string + } + + // get list of commentID for all user's comment + comments := []commentInfo{} + err = bdb.View(func(tx *bolt.Tx) error { + postsBkt := tx.Bucket([]byte(postsBucketName)) + for _, postInfo := range posts { + postBkt := postsBkt.Bucket([]byte(postInfo.URL)) + err = postsBkt.ForEach(func(k []byte, v []byte) error { + comment, err := b.load(postBkt, k) + if err != nil { + return errors.Wrapf(err, "can't load key %s from bucket %s", k, postInfo.URL) + } + if comment.User.ID == userID { + comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID}) + } + return nil + }) + + if err != nil { + return errors.Wrapf(err, "failed to delete comments from %s", postInfo.URL) + } + } + return nil + }) + if err != nil { + return errors.Wrapf(err, "failed to collect list of all comments for deletion") + } + + // delete collected comments + for _, ci := range comments { + b.Delete(ci.locator, ci.commentID, store.HardDelete) + } + + // delete user bucket + err = bdb.Update(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + if usersBkt != nil { + if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { + return errors.Wrapf(err, "failed to delete user bucker for %s", userID) + } + } + return nil + }) + + return nil +} + +// SetBlock blocks/unblocks user for given site +func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error { + + bdb, err := b.db(siteID) + if err != nil { + return err + } + + return bdb.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + switch status { + case true: + if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { + return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) + } + case false: + if e := bucket.Delete([]byte(userID)); e != nil { + return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) + } + } + return nil + }) +} + +// IsBlocked checks if user blocked +func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { + + bdb, err := b.db(siteID) + if err != nil { + return false + } + + _ = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + blocked = bucket.Get([]byte(userID)) != nil + return nil + }) + return blocked +} + +// Blocked get lists of blocked users for given site +// bucket uses userID: +func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { + users = []store.BlockedUser{} + bdb, err := b.db(siteID) + if err != nil { + return nil, err + } + + err = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + return bucket.ForEach(func(k []byte, v []byte) error { + ts, e := time.ParseInLocation(tsNano, string(v), time.Local) + if e != nil { + return errors.Wrap(e, "can't parse block ts") + } + + // get user name from comment user section + userName := "" + if userComments, _, e := b.User(siteID, string(k), 1); e == nil && len(userComments) > 0 { + userName = userComments[0].User.Name + } + + users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts}) + return nil + }) + }) + + return users, err +} diff --git a/app/store/engine/bolt_test.go b/app/store/engine/bolt_test.go index 39195462..ab3cdbc3 100644 --- a/app/store/engine/bolt_test.go +++ b/app/store/engine/bolt_test.go @@ -41,7 +41,7 @@ func TestBoltDB_Delete(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 2, len(res), "initially 2 comments") - err = b.Delete(loc, res[0].ID) + err = b.Delete(loc, res[0].ID, store.SoftDelete) assert.Nil(t, err) res, err = b.Find(loc, "time") @@ -49,6 +49,8 @@ func TestBoltDB_Delete(t *testing.T) { assert.Equal(t, 2, len(res)) assert.Equal(t, "", res[0].Text) assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) + assert.Equal(t, "some text2", res[1].Text) assert.False(t, res[1].Deleted) @@ -56,14 +58,34 @@ func TestBoltDB_Delete(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - err = b.Delete(loc, "123456") + err = b.Delete(loc, "123456", store.SoftDelete) assert.NotNil(t, err) loc.SiteID = "bad" - err = b.Delete(loc, res[0].ID) + err = b.Delete(loc, res[0].ID, store.SoftDelete) assert.EqualError(t, err, `site "bad" not found`) } +func TestBoltDB_DeleteHard(t *testing.T) { + defer os.Remove(testDb) + b := prep(t) + + loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} + res, err := b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + err = b.Delete(loc, res[0].ID, store.HardDelete) + assert.Nil(t, err) + + res, err = b.Find(loc, "time") + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "", res[0].Text) + assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) +} + func TestBoltDB_DeleteAll(t *testing.T) { defer os.Remove(testDb) b := prep(t) diff --git a/app/store/engine/engine.go b/app/store/engine/engine.go index 4a520268..10caa24f 100644 --- a/app/store/engine/engine.go +++ b/app/store/engine/engine.go @@ -31,11 +31,12 @@ type Accessor interface { // Admin defines all store ops avail for admin only type Admin interface { - Delete(locator store.Locator, commentID string) error // delete comment by id - DeleteAll(siteID string) error // delete all data from site - SetBlock(siteID string, userID string, status bool) error // block or unblock user - IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users + Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id + DeleteAll(siteID string) error // delete all data from site + DeleteUser(siteID string, userID string) error // remove all comments from user + SetBlock(siteID string, userID string, status bool) error // block or unblock user + IsBlocked(siteID string, userID string) bool // check if user blocked + Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users } // sortComments is for engines can't sort data internally