diff --git a/README.md b/README.md index bb3cb187..ccb3f0cd 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Sort can be `time` or `score`. Supported sort order with prefix -/+, i.e. `-time }{} ``` - `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}` -- `GET /api/v1/list?site=site-id` - list commented posts, returns array or `PostInfo` +- `GET /api/v1/list?site=site-id&limit=5` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts ```go type PostInfo struct { URL string `json:"url"` diff --git a/app/migrator/disqus_test.go b/app/migrator/disqus_test.go index e2a7cd9f..713aad4e 100644 --- a/app/migrator/disqus_test.go +++ b/app/migrator/disqus_test.go @@ -33,7 +33,7 @@ func TestDisqus_Import(t *testing.T) { assert.Equal(t, "disqus_google-74b9e7568ef6860e93862c5d77590123", c.User.ID) assert.Equal(t, "89.89.89.139", c.User.IP) - posts, err := dataStore.List("test") + posts, err := dataStore.List("test", 0) assert.Nil(t, err) assert.Equal(t, 2, len(posts), "2 posts") diff --git a/app/migrator/remark.go b/app/migrator/remark.go index 64e80dae..25d3838a 100644 --- a/app/migrator/remark.go +++ b/app/migrator/remark.go @@ -19,14 +19,15 @@ type Remark struct { // Export all comments to writer as json strings. Each comment is one string, separated by "\n" func (r *Remark) Export(w io.Writer, siteID string) error { - topics, err := r.DataStore.List(siteID) + topics, err := r.DataStore.List(siteID, 0) if err != nil { return err } log.Printf("[DEBUG] exporting %d topics", len(topics)) commentsCount := 0 - for _, topic := range topics { + for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction + topic := topics[i] comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") if err != nil { return err diff --git a/app/rest/api/admin.go b/app/rest/api/admin.go index 62d9efae..4a31922f 100644 --- a/app/rest/api/admin.go +++ b/app/rest/api/admin.go @@ -138,7 +138,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res [] // process blocked users if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) { - c.Mask() + c.SetDeleted() c.User.Blocked = true } diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index a93b4b4a..db35eef9 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -375,12 +375,18 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, JSON{"count": count, "locator": locator}) } -// GET /list?site=siteID - list posts with comments +// GET /list?site=siteID&limit=50 - list posts with comments func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") + limit := 0 + limitStr := r.URL.Query().Get("limit") + if v, err := strconv.Atoi(limitStr); err == nil { + limit = v + } + data, err := s.Cache.Get(rest.URLKey(r), 8*time.Hour, func() ([]byte, error) { - posts, e := s.DataService.List(siteID) + posts, e := s.DataService.List(siteID, limit) if e != nil { return nil, e } diff --git a/app/rest/api/rest_test.go b/app/rest/api/rest_test.go index 87b0919f..8071dfdc 100644 --- a/app/rest/api/rest_test.go +++ b/app/rest/api/rest_test.go @@ -350,7 +350,7 @@ func TestServer_List(t *testing.T) { pi := []store.PostInfo{} err := json.Unmarshal([]byte(body), &pi) assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3}, {URL: "https://radio-t.com/blah2", Count: 2}}, pi) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 2}, {URL: "https://radio-t.com/blah1", Count: 3}}, pi) } func prep(t *testing.T) (srv *Rest, port int) { diff --git a/app/store/bolt.go b/app/store/bolt.go index 50a337f0..7442bc16 100644 --- a/app/store/bolt.go +++ b/app/store/bolt.go @@ -14,13 +14,14 @@ import ( ) // BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe. -// there are 4 types of buckets: +// there are 5 types of top-level buckets: // - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment // - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID // value is not full comment but a reference combined from post-url+commentID // - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value // is a nested bucket named userID with kv as ts:reference // - blocking info sits in "block" bucket. Key is userID, value - ts +// - counts per post to keep number of comments. Key is post url, value - count type BoltDB struct { dbs map[string]*bolt.DB } @@ -75,7 +76,7 @@ func NewBoltDB(sites ...BoltSite) (*BoltDB, error) { return &result, nil } -// Create saves new comment to store +// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket func (b *BoltDB) Create(comment Comment) (commentID string, err error) { // fill ID and time if empty @@ -143,7 +144,9 @@ func (b *BoltDB) Create(comment Comment) (commentID string, err error) { return comment.ID, err } -// Delete removes comment, by locator from the store +// Delete removes comment, by locator from the store. +// Posts collection only sets status to deleted and clear fileds in order to prevent breaking trees of replies. +// From last bucket removed for real. func (b *BoltDB) Delete(locator Locator, commentID string) error { bdb, err := b.db(locator.SiteID) @@ -163,8 +166,7 @@ func (b *BoltDB) Delete(locator Locator, commentID string) error { return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) } // set deleted status and clear fields - comment.Mask() - comment.Deleted = true + comment.SetDeleted() if err := b.save(postBkt, []byte(commentID), comment); err != nil { return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) @@ -337,7 +339,8 @@ func (b *BoltDB) Blocked(siteID string) (users []BlockedUser, err error) { } // List returns list of all commented posts with counters -func (b BoltDB) List(siteID string) (list []PostInfo, err error) { +// uses count bucket to get number of comments +func (b BoltDB) List(siteID string, limit int) (list []PostInfo, err error) { bdb, err := b.db(siteID) if err != nil { @@ -346,15 +349,20 @@ func (b BoltDB) List(siteID string) (list []PostInfo, err error) { err = bdb.View(func(tx *bolt.Tx) error { postsBkt := tx.Bucket([]byte(postsBucketName)) - return postsBkt.ForEach(func(name []byte, _ []byte) error { - postURL := string(name) + + c := postsBkt.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + postURL := string(k) count, e := b.count(tx, postURL, 0) if e != nil { return e } list = append(list, PostInfo{URL: postURL, Count: count}) - return nil - }) + if limit > 0 && len(list) >= limit { + break + } + } + return nil }) return list, err @@ -380,11 +388,11 @@ func (b *BoltDB) User(siteID string, userID string) (comments []Comment, totalCo } c := userIDBkt.Cursor() - totalComments = userIDBkt.Stats().KeyN + totalComments = 0 for k, v := c.Last(); k != nil; k, v = c.Prev() { - commentRefs = append(commentRefs, string(v)) - if len(commentRefs) > userLimit { - break + totalComments++ + if len(commentRefs) <= userLimit { + commentRefs = append(commentRefs, string(v)) } } return nil @@ -502,7 +510,7 @@ func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment Comment, err error) } // count adds val to counts key postURL. val can be negative to substruct. if val 0 can be used as accessor -// it uses seprate counts bucket because boltdb Stat call is very slow +// it uses separate counts bucket because boltdb Stat call is very slow func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { btoi := func(v []byte) int { @@ -522,8 +530,8 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { if val == 0 { return btoi(countVal), nil } - newVal := itob(btoi(countVal) + val) - return btoi(newVal), countBkt.Put([]byte(postURL), newVal) + updatedCount := btoi(countVal) + val + return updatedCount, countBkt.Put([]byte(postURL), itob(updatedCount)) } func (b *BoltDB) db(siteID string) (*bolt.DB, error) { diff --git a/app/store/bolt_test.go b/app/store/bolt_test.go index de17618e..58785e2f 100644 --- a/app/store/bolt_test.go +++ b/app/store/bolt_test.go @@ -160,9 +160,13 @@ func TestBoltDB_List(t *testing.T) { _, err := b.Create(comment) assert.Nil(t, err) - res, err := b.List("radio-t") + res, err := b.List("radio-t", 0) assert.Nil(t, err) - assert.Equal(t, []PostInfo{{URL: "https://radio-t.com", Count: 2}, {URL: "https://radio-t.com/2", Count: 1}}, res) + assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}, {URL: "https://radio-t.com", Count: 2}}, res) + + res, err = b.List("radio-t", 1) + assert.Nil(t, err) + assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) } func TestBoltDB_GetForUser(t *testing.T) { diff --git a/app/store/comment.go b/app/store/comment.go index 394870c9..8738049c 100644 --- a/app/store/comment.go +++ b/app/store/comment.go @@ -71,12 +71,13 @@ func (c *Comment) Sanitize() { c.Text = strings.Replace(c.Text, "\t", "", -1) } -// Mask clears comment info, reset to "Deleted/Blocked" -func (c *Comment) Mask() { +// SetDeleted clears comment info, reset to "Deleted/Blocked" +func (c *Comment) SetDeleted() { c.Text = "this comment was deleted" c.Score = 0 c.Votes = map[string]bool{} c.Edit = nil + c.Deleted = true } // NotifUser holds id and destination for notifiable user diff --git a/app/store/store.go b/app/store/store.go index 76d4b376..93fb901f 100644 --- a/app/store/store.go +++ b/app/store/store.go @@ -22,7 +22,7 @@ type Accessor interface { Last(siteID string, max int) ([]Comment, error) // last comments for given site, sorted by time User(siteID string, userID string) ([]Comment, int, error) // comments by user, sorted by time Count(locator Locator) (int, error) // number of comments for the post - List(siteID string) ([]PostInfo, error) // list of commented posts + List(siteID string, limit int) ([]PostInfo, error) // list of commented posts } // Admin defines all store ops avail for admin only