mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

list with limits

Browse Source
This commit is contained in:
Umputun
2018-02-19 12:19:16 -06:00
parent dc235d6aad
commit 0366e99d89
10 changed files with 50 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -190,7 +190,7 @@ Sort can be `time` or `score`. Supported sort order with prefix -/+, i.e. `-time
}{}
```
- `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
- `GET /api/v1/list?site=site-id` - list commented posts, returns array or `PostInfo`
- `GET /api/v1/list?site=site-id&limit=5` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
```go
type PostInfo struct {
URL string `json:"url"`
app/migrator/disqus_test.go
+1 -1
View File
@@ -33,7 +33,7 @@ func TestDisqus_Import(t *testing.T) {
assert.Equal(t, "disqus_google-74b9e7568ef6860e93862c5d77590123", c.User.ID)
assert.Equal(t, "89.89.89.139", c.User.IP)
posts, err := dataStore.List("test")
posts, err := dataStore.List("test", 0)
assert.Nil(t, err)
assert.Equal(t, 2, len(posts), "2 posts")
app/migrator/remark.go
+3 -2
View File
@@ -19,14 +19,15 @@ type Remark struct {
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
func (r *Remark) Export(w io.Writer, siteID string) error {
topics, err := r.DataStore.List(siteID)
topics, err := r.DataStore.List(siteID, 0)
if err != nil {
return err
}
log.Printf("[DEBUG] exporting %d topics", len(topics))
commentsCount := 0
for _, topic := range topics {
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
topic := topics[i]
comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
if err != nil {
return err
app/rest/api/admin.go
+1 -1
View File
@@ -138,7 +138,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []
// process blocked users
if a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID) {
c.Mask()
c.SetDeleted()
c.User.Blocked = true
}
app/rest/api/rest.go
+8 -2
View File
@@ -375,12 +375,18 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
render.JSON(w, r, JSON{"count": count, "locator": locator})
}
// GET /list?site=siteID - list posts with comments
// GET /list?site=siteID&limit=50 - list posts with comments
func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
limit := 0
limitStr := r.URL.Query().Get("limit")
if v, err := strconv.Atoi(limitStr); err == nil {
limit = v
}
data, err := s.Cache.Get(rest.URLKey(r), 8*time.Hour, func() ([]byte, error) {
posts, e := s.DataService.List(siteID)
posts, e := s.DataService.List(siteID, limit)
if e != nil {
return nil, e
}
app/rest/api/rest_test.go
+1 -1
View File
@@ -350,7 +350,7 @@ func TestServer_List(t *testing.T) {
pi := []store.PostInfo{}
err := json.Unmarshal([]byte(body), &pi)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3}, {URL: "https://radio-t.com/blah2", Count: 2}}, pi)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah2", Count: 2}, {URL: "https://radio-t.com/blah1", Count: 3}}, pi)
}
func prep(t *testing.T) (srv *Rest, port int) {
app/store/bolt.go
+25 -17
View File
@@ -14,13 +14,14 @@ import (
)
// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
// there are 4 types of buckets:
// there are 5 types of top-level buckets:
// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
// value is not full comment but a reference combined from post-url+commentID
// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
// is a nested bucket named userID with kv as ts:reference
// - blocking info sits in "block" bucket. Key is userID, value - ts
// - counts per post to keep number of comments. Key is post url, value - count
type BoltDB struct {
dbs map[string]*bolt.DB
}
@@ -75,7 +76,7 @@ func NewBoltDB(sites ...BoltSite) (*BoltDB, error) {
return &result, nil
}
// Create saves new comment to store
// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
func (b *BoltDB) Create(comment Comment) (commentID string, err error) {
// fill ID and time if empty
@@ -143,7 +144,9 @@ func (b *BoltDB) Create(comment Comment) (commentID string, err error) {
return comment.ID, err
}
// Delete removes comment, by locator from the store
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fileds in order to prevent breaking trees of replies.
// From last bucket removed for real.
func (b *BoltDB) Delete(locator Locator, commentID string) error {
bdb, err := b.db(locator.SiteID)
@@ -163,8 +166,7 @@ func (b *BoltDB) Delete(locator Locator, commentID string) error {
return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
}
// set deleted status and clear fields
comment.Mask()
comment.Deleted = true
comment.SetDeleted()
if err := b.save(postBkt, []byte(commentID), comment); err != nil {
return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
@@ -337,7 +339,8 @@ func (b *BoltDB) Blocked(siteID string) (users []BlockedUser, err error) {
}
// List returns list of all commented posts with counters
func (b BoltDB) List(siteID string) (list []PostInfo, err error) {
// uses count bucket to get number of comments
func (b BoltDB) List(siteID string, limit int) (list []PostInfo, err error) {
bdb, err := b.db(siteID)
if err != nil {
@@ -346,15 +349,20 @@ func (b BoltDB) List(siteID string) (list []PostInfo, err error) {
err = bdb.View(func(tx *bolt.Tx) error {
postsBkt := tx.Bucket([]byte(postsBucketName))
return postsBkt.ForEach(func(name []byte, _ []byte) error {
postURL := string(name)
c := postsBkt.Cursor()
for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
postURL := string(k)
count, e := b.count(tx, postURL, 0)
if e != nil {
return e
}
list = append(list, PostInfo{URL: postURL, Count: count})
return nil
})
if limit > 0 && len(list) >= limit {
break
}
}
return nil
})
return list, err
@@ -380,11 +388,11 @@ func (b *BoltDB) User(siteID string, userID string) (comments []Comment, totalCo
}
c := userIDBkt.Cursor()
totalComments = userIDBkt.Stats().KeyN
totalComments = 0
for k, v := c.Last(); k != nil; k, v = c.Prev() {
commentRefs = append(commentRefs, string(v))
if len(commentRefs) > userLimit {
break
totalComments++
if len(commentRefs) <= userLimit {
commentRefs = append(commentRefs, string(v))
}
}
return nil
@@ -502,7 +510,7 @@ func (b *BoltDB) load(bkt *bolt.Bucket, key []byte) (comment Comment, err error)
}
// count adds val to counts key postURL. val can be negative to substruct. if val 0 can be used as accessor
// it uses seprate counts bucket because boltdb Stat call is very slow
// it uses separate counts bucket because boltdb Stat call is very slow
func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
btoi := func(v []byte) int {
@@ -522,8 +530,8 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
if val == 0 {
return btoi(countVal), nil
}
newVal := itob(btoi(countVal) + val)
return btoi(newVal), countBkt.Put([]byte(postURL), newVal)
updatedCount := btoi(countVal) + val
return updatedCount, countBkt.Put([]byte(postURL), itob(updatedCount))
}
func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
app/store/bolt_test.go
+6 -2
View File
@@ -160,9 +160,13 @@ func TestBoltDB_List(t *testing.T) {
_, err := b.Create(comment)
assert.Nil(t, err)
res, err := b.List("radio-t")
res, err := b.List("radio-t", 0)
assert.Nil(t, err)
assert.Equal(t, []PostInfo{{URL: "https://radio-t.com", Count: 2}, {URL: "https://radio-t.com/2", Count: 1}}, res)
assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}, {URL: "https://radio-t.com", Count: 2}}, res)
res, err = b.List("radio-t", 1)
assert.Nil(t, err)
assert.Equal(t, []PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res)
}
func TestBoltDB_GetForUser(t *testing.T) {
app/store/comment.go
+3 -2
View File
@@ -71,12 +71,13 @@ func (c *Comment) Sanitize() {
c.Text = strings.Replace(c.Text, "\t", "", -1)
}
// Mask clears comment info, reset to "Deleted/Blocked"
func (c *Comment) Mask() {
// SetDeleted clears comment info, reset to "Deleted/Blocked"
func (c *Comment) SetDeleted() {
c.Text = "this comment was deleted"
c.Score = 0
c.Votes = map[string]bool{}
c.Edit = nil
c.Deleted = true
}
// NotifUser holds id and destination for notifiable user
app/store/store.go
+1 -1
View File
@@ -22,7 +22,7 @@ type Accessor interface {
Last(siteID string, max int) ([]Comment, error) // last comments for given site, sorted by time
User(siteID string, userID string) ([]Comment, int, error) // comments by user, sorted by time
Count(locator Locator) (int, error) // number of comments for the post
List(siteID string) ([]PostInfo, error) // list of commented posts
List(siteID string, limit int) ([]PostInfo, error) // list of commented posts
}
// Admin defines all store ops avail for admin only