diff --git a/README.md b/README.md
index 237edd29..4134ff4a 100644
--- a/README.md
+++ b/README.md
@@ -581,7 +581,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
}{}
```
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
* `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
* `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
```go
diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go
index 334c00e9..3f3db451 100644
--- a/backend/app/migrator/disqus_test.go
+++ b/backend/app/migrator/disqus_test.go
@@ -26,7 +26,7 @@ func TestDisqus_Import(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 4, size)
- last, err := dataStore.Last("test", 10)
+ last, err := dataStore.Last("test", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 4, len(last), "4 comments imported")
diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go
index ff3bf1ca..c004eb1c 100644
--- a/backend/app/migrator/migrator_test.go
+++ b/backend/app/migrator/migrator_test.go
@@ -4,6 +4,7 @@ import (
"io/ioutil"
"os"
"testing"
+ "time"
bolt "github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
@@ -35,7 +36,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 4, size)
- last, err := dataStore.Last("test", 10)
+ last, err := dataStore.Last("test", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 4, len(last), "4 comments imported")
}
@@ -61,7 +62,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 3, size)
- last, err := dataStore.Last("test", 10)
+ last, err := dataStore.Last("test", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 3, len(last), "3 comments imported")
}
@@ -91,7 +92,7 @@ func TestMigrator_ImportNative(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, size)
- last, err := dataStore.Last("radio-t", 10)
+ last, err := dataStore.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(last), "2 comments imported")
}
diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go
index f6ec722c..7a0acbd4 100644
--- a/backend/app/migrator/native_test.go
+++ b/backend/app/migrator/native_test.go
@@ -82,7 +82,7 @@ func TestNative_Import(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, size)
- comments, err := b.Last("radio-t", 10)
+ comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(comments))
assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go
index e9d64e9e..001fbffd 100644
--- a/backend/app/migrator/wordpress_test.go
+++ b/backend/app/migrator/wordpress_test.go
@@ -27,7 +27,7 @@ func TestWordPress_Import(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 3, size)
- last, err := dataStore.Last(siteID, 10)
+ last, err := dataStore.Last(siteID, 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 3, len(last), "3 comments imported")
diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go
index 42ba116d..2a795514 100644
--- a/backend/app/rest/api/rest_public.go
+++ b/backend/app/rest/api/rest_public.go
@@ -9,6 +9,7 @@ import (
"path"
"strconv"
"strings"
+ "time"
"github.com/go-chi/chi"
"github.com/go-chi/render"
@@ -117,7 +118,8 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
}
}
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
+// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
+// limited with "since" param
func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get last comments for %s", siteID)
@@ -127,9 +129,20 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
limit = 0
}
+ sinceTime := time.Time{}
+ since := r.URL.Query().Get("since")
+ if since != "" {
+ unixTS, err := strconv.ParseInt(since, 10, 64)
+ if err != nil {
+ rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+ return
+ }
+ sinceTime = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+ }
+
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
- comments, e := s.DataService.Last(siteID, limit)
+ comments, e := s.DataService.Last(siteID, limit, sinceTime)
if e != nil {
return nil, e
}
diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go
index 0e9b6444..3334ccc6 100644
--- a/backend/app/rest/api/rest_public_test.go
+++ b/backend/app/rest/api/rest_public_test.go
@@ -209,8 +209,11 @@ func TestRest_Last(t *testing.T) {
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
// add 3 comments
+ ts1 := time.Now().UnixNano() / 1000000
addComment(t, c1, ts)
id1 := addComment(t, c1, ts)
+ time.Sleep(10 * time.Millisecond)
+ ts2 := time.Now().UnixNano() / 1000000
id2 := addComment(t, c2, ts)
res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
@@ -222,6 +225,23 @@ func TestRest_Last(t *testing.T) {
assert.Equal(t, id1, comments[1].ID)
assert.Equal(t, id2, comments[0].ID)
+ res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts1))
+ assert.Equal(t, 200, code)
+ comments = []store.Comment{}
+ err = json.Unmarshal([]byte(res), &comments)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(comments), "should have 2 comments")
+ assert.Equal(t, id1, comments[1].ID)
+ assert.Equal(t, id2, comments[0].ID)
+
+ res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts2))
+ assert.Equal(t, 200, code)
+ comments = []store.Comment{}
+ err = json.Unmarshal([]byte(res), &comments)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(comments), "should have 1 comments")
+ assert.Equal(t, id2, comments[0].ID)
+
res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
assert.Equal(t, 200, code)
err = json.Unmarshal([]byte(res), &comments)
diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go
index ce70a642..62d51b67 100644
--- a/backend/app/rest/api/rss.go
+++ b/backend/app/rest/api/rss.go
@@ -60,7 +60,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
- comments, e := s.DataService.Last(siteID, maxRssItems)
+ comments, e := s.DataService.Last(siteID, maxRssItems, time.Time{})
if e != nil {
return nil, e
}
@@ -94,7 +94,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
userName := ""
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() (res []byte, e error) {
- comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+ comments, e := s.DataService.Last(siteID, maxLastCommentsReply, time.Time{})
if e != nil {
return nil, errors.Wrap(e, "can't get last comments")
}
diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 862ab0e3..1a555937 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -1,6 +1,7 @@
package engine
import (
+ "bytes"
"encoding/json"
"fmt"
"strings"
@@ -169,7 +170,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
}
// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
+func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
comments = []store.Comment{}
@@ -185,7 +186,16 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
err = bdb.View(func(tx *bolt.Tx) error {
lastBkt := tx.Bucket([]byte(lastBucketName))
c := lastBkt.Cursor()
+
for k, v := c.Last(); k != nil; k, v = c.Prev() {
+
+ if !since.IsZero() {
+ // stop if reached "since" ts
+ tsSince := []byte(since.Format(tsNano))
+ if bytes.Compare(k, tsSince) <= 0 {
+ break
+ }
+ }
url, commentID, e := b.parseRef(v)
if e != nil {
return e
diff --git a/backend/app/store/engine/bolt_accessor_test.go b/backend/app/store/engine/bolt_accessor_test.go
index fe5995e7..1bf900d6 100644
--- a/backend/app/store/engine/bolt_accessor_test.go
+++ b/backend/app/store/engine/bolt_accessor_test.go
@@ -111,20 +111,42 @@ func TestBoltDB_Last(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
- res, err = b.Last("radio-t", 1)
+ res, err = b.Last("radio-t", 1, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
- _, err = b.Last("bad", 0)
+ _, err = b.Last("bad", 0, time.Time{})
assert.EqualError(t, err, `site "bad" not found`)
}
+func TestBoltDB_LastSince(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+ res, err := b.Last("radio-t", 0, ts)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ ts = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+ res, err = b.Last("radio-t", 0, ts)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ ts = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+ res, err = b.Last("radio-t", 0, ts)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
func TestBoltDB_Count(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
diff --git a/backend/app/store/engine/bolt_admin_test.go b/backend/app/store/engine/bolt_admin_test.go
index b2196f43..06899b3b 100644
--- a/backend/app/store/engine/bolt_admin_test.go
+++ b/backend/app/store/engine/bolt_admin_test.go
@@ -37,7 +37,7 @@ func TestBoltAdmin_Delete(t *testing.T) {
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
- comments, err := b.Last("radio-t", 10)
+ comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
@@ -91,7 +91,7 @@ func TestBoltAdmin_DeleteAll(t *testing.T) {
err = b.DeleteAll("radio-t")
assert.Nil(t, err)
- comments, err := b.Last("radio-t", 10)
+ comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
@@ -125,7 +125,7 @@ func TestBoltAdmin_DeleteUser(t *testing.T) {
_, err = b.User("radio-t", "user1", 5, 0)
assert.EqualError(t, err, "no comments for user user1 in store")
- comments, err := b.Last("radio-t", 10)
+ comments, err := b.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go
index 5ddbeee4..dbbdd654 100644
--- a/backend/app/store/engine/engine.go
+++ b/backend/app/store/engine/engine.go
@@ -28,17 +28,17 @@ type UserRequest struct {
// Accessor defines all usual access ops avail for regular user
type Accessor interface {
- Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
- Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id
- Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only
- Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator
- Last(siteID string, limit int) ([]store.Comment, error) // last comments for given site, sorted by time
- User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
- UserCount(siteID, userID string) (int, error) // comments count by user
- Count(locator store.Locator) (int, error) // number of comments for the post
- List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
- Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
- Close() error // close/stop engine
+ Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
+ Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id
+ Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only
+ Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator
+ Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time
+ User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
+ UserCount(siteID, userID string) (int, error) // comments count by user
+ Count(locator store.Locator) (int, error) // number of comments for the post
+ List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
+ Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
+ Close() error // close/stop engine
}
// Admin defines all store ops avail for admin only
diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go
index 13e0a12b..3520711d 100644
--- a/backend/app/store/engine/mongo.go
+++ b/backend/app/store/engine/mongo.go
@@ -92,13 +92,16 @@ func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
}
// Last returns up to max last comments for given siteID
-func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
+func (m *Mongo) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
comments = []store.Comment{}
if max > lastLimit || max == 0 {
max = lastLimit
}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": siteID, "delete": false}
+ if !since.IsZero() {
+ query["time"] = bson.M{"$gt": since}
+ }
return coll.Find(query).Sort("-time").Limit(max).All(&comments)
})
return comments, err
diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine/mongo_test.go
index 797d2e76..5b3f4b27 100644
--- a/backend/app/store/engine/mongo_test.go
+++ b/backend/app/store/engine/mongo_test.go
@@ -90,12 +90,22 @@ func TestMongo_Last(t *testing.T) {
if skip {
return
}
- res, err := m.Last("radio-t", 0)
+ res, err := m.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
- res, err = m.Last("radio-t", 1)
+ res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local))
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local))
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ res, err = m.Last("radio-t", 1, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
@@ -428,7 +438,7 @@ func TestMongo_Delete(t *testing.T) {
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
- comments, err := m.Last("radio-t", 10)
+ comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
@@ -478,7 +488,7 @@ func TestMongo_DeleteAll(t *testing.T) {
err = m.DeleteAll("radio-t")
assert.Nil(t, err)
- comments, err := m.Last("radio-t", 10)
+ comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
@@ -510,7 +520,7 @@ func TestMongo_DeleteUser(t *testing.T) {
assert.Nil(t, err, "no comments for user user1 in store")
assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
- comments, err := m.Last("radio-t", 10)
+ comments, err := m.Last("radio-t", 10, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
}
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 36ad92e7..0d6ee63f 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -295,7 +295,7 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
return true
}
- comments, err := s.Last(comment.Locator.SiteID, maxLastCommentsReply)
+ comments, err := s.Last(comment.Locator.SiteID, maxLastCommentsReply, time.Time{})
if err != nil {
log.Printf("[WARN] can't get last comments for reply check, %v", err)
return false
diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index dda0568b..52f8d041 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -97,7 +97,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
res, err := b.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
assert.NoError(t, err)
t.Logf("%+v", res)
- assert.Equal(t, "Радио-Т 630", res.PostTitle)
+ assert.Equal(t, "Радио-Т 630 — Радио-Т Подкаст", res.PostTitle)
comment.PostTitle = "post blah"
id, err = b.Create(comment)
@@ -174,7 +174,7 @@ func TestService_Vote(t *testing.T) {
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
@@ -195,7 +195,7 @@ func TestService_Vote(t *testing.T) {
assert.NotNil(t, err, "double-voting rejected")
assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted"))
- res, err = b.Last("radio-t", 0)
+ res, err = b.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, 1, res[0].Score)
@@ -204,7 +204,7 @@ func TestService_Vote(t *testing.T) {
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
assert.Nil(t, err, "vote reset")
- res, err = b.Last("radio-t", 0)
+ res, err = b.Last("radio-t", 0, time.Time{})
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, 0, res[0].Score)
@@ -250,7 +250,7 @@ func TestService_VoteAggressive(t *testing.T) {
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
require.Nil(t, err)
t.Logf("%+v", res[0])
assert.Equal(t, 3, len(res))
@@ -271,7 +271,7 @@ func TestService_VoteAggressive(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Last("radio-t", 0)
+ res, err = b.Last("radio-t", 0, time.Time{})
require.NoError(t, err)
t.Logf("%+v", res[0])
@@ -290,7 +290,7 @@ func TestService_VoteAggressive(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Last("radio-t", 0)
+ res, err = b.Last("radio-t", 0, time.Time{})
require.NoError(t, err)
assert.Equal(t, 3, len(res))
t.Logf("%+v %d", res[0], res[0].Score)
@@ -309,7 +309,7 @@ func TestService_VoteConcurrent(t *testing.T) {
}
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
require.Nil(t, err)
// concurrent vote +1 as multiple users for the same comment
@@ -324,7 +324,7 @@ func TestService_VoteConcurrent(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Last("radio-t", 0)
+ res, err = b.Last("radio-t", 0, time.Time{})
require.NoError(t, err)
assert.Equal(t, 100, res[0].Score, "should have 100 score")
assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
@@ -370,7 +370,7 @@ func TestService_VoteControversy(t *testing.T) {
assert.InDelta(t, 1.73, c.Controversy, 0.01)
// check if stored
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
require.NoError(t, err)
assert.Equal(t, 1, res[0].Score, "should have 1 score")
assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
@@ -404,7 +404,7 @@ func TestService_Pin(t *testing.T) {
defer teardown(t)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
@@ -428,7 +428,7 @@ func TestService_EditComment(t *testing.T) {
defer teardown(t)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
@@ -455,7 +455,7 @@ func TestService_DeleteComment(t *testing.T) {
defer teardown(t)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
@@ -474,7 +474,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
defer teardown(t)
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
@@ -491,7 +491,7 @@ func TestService_EditCommentReplyFailed(t *testing.T) {
defer teardown(t)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Last("radio-t", 0)
+ res, err := b.Last("radio-t", 0, time.Time{})
t.Logf("%+v", res[1])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
diff --git a/backend/app/store/service/title.go b/backend/app/store/service/title.go
index 12f5bfbb..bfefbc47 100644
--- a/backend/app/store/service/title.go
+++ b/backend/app/store/service/title.go
@@ -3,6 +3,7 @@ package service
import (
"io"
"net/http"
+ "strings"
"time"
"github.com/go-pkgz/lcw"
@@ -85,7 +86,10 @@ func (t *TitleExtractor) isTitleElement(n *html.Node) bool {
func (t *TitleExtractor) traverse(n *html.Node) (string, bool) {
if t.isTitleElement(n) {
- return n.FirstChild.Data, true
+ title := n.FirstChild.Data
+ title = strings.Replace(title, "\n", "", -1)
+ title = strings.TrimSpace(title)
+ return title, true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
diff --git a/backend/app/store/service/title_test.go b/backend/app/store/service/title_test.go
index aadd4072..16a31515 100644
--- a/backend/app/store/service/title_test.go
+++ b/backend/app/store/service/title_test.go
@@ -24,7 +24,8 @@ func TestTitle_GetTitle(t *testing.T) {
title string
}{
{`blah 123 2222`, true, "blah 123"},
- {`blah 123 `, true, "blah 123 "},
+ {`<html><title>blah 123 `, true, "blah 123"},
+ {"<html><title>\n\n blah 123 \n ", true, "blah 123"},
{`<html><body> 2222</body></html>`, false, ""},
}
@@ -44,7 +45,7 @@ func TestTitle_Get(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.String() == "/good" {
atomic.AddInt32(&hits, 1)
- _, err := w.Write([]byte("<html><title>blah 123 2222"))
+ _, err := w.Write([]byte("\n\n blah 123\n 2222"))
assert.NoError(t, err)
return
}