mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Find since (#388)

Browse Source 
* support since param in bolt find for post
* add find with since to store service
* add since param to find request, plain only
This commit is contained in:
Umputun
2019-07-25 13:58:22 -05:00
committed by GitHub
parent 5431f80e84
commit eb3589869d
5 changed files with 99 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/rest/api/rest_public.go
+47 -34
View File
@@ -16,6 +16,7 @@ import (
log "github.com/go-pkgz/lgr"
R "github.com/go-pkgz/rest"
"github.com/go-pkgz/rest/cache"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
@@ -36,7 +37,7 @@ type public struct {
type pubStore interface {
Create(comment store.Comment) (commentID string, err error)
Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
UserCount(siteID, userID string) (int, error)
@@ -49,7 +50,7 @@ type pubStore interface {
Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
}
// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]&view=[user|all]
// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
// find comments for given post. Returns in tree or plain formats, sorted
func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -59,17 +60,27 @@ func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
}
view := r.URL.Query().Get("view")
log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
since, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
return
}
format := r.URL.Query().Get("format")
if format == "tree" {
since = time.Time{} // since doesn't make sense for tree
}
log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v", locator, sort, format, since)
key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
data, err := s.cache.Get(key, func() ([]byte, error) {
comments, e := s.dataService.Find(locator, sort, rest.GetUserOrEmpty(r))
comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
if e != nil {
comments = []store.Comment{} // error should clear comments and continue for post info
}
comments = s.applyView(comments, view)
var b []byte
switch r.URL.Query().Get("format") {
switch format {
case "tree":
tree := service.MakeTree(comments, sort, s.readOnlyAge)
if tree.Nodes == nil { // eliminate json nil serialization
@@ -152,15 +163,10 @@ func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
sinceTs := time.Time{}
since := r.URL.Query().Get("since")
if since != "" {
unixTS, e := strconv.ParseInt(since, 10, 64)
if e != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, e, "can't translate since parameter", rest.ErrDecode)
return
}
sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
sinceTs, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
return
}
fn := func() steamEventFn {
@@ -191,8 +197,8 @@ func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
}
}
if err := s.streamer.Activate(r.Context(), fn, w); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't stream", rest.ErrInternal)
if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
}
}
@@ -207,14 +213,10 @@ func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
limit = 0
}
sinceTime := time.Time{}
if since := r.URL.Query().Get("since"); since != "" {
unixTS, e := strconv.ParseInt(since, 10, 64)
if e != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, e, "can't translate since parameter", rest.ErrDecode)
return
}
sinceTime = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
sinceTime, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
return
}
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
@@ -243,14 +245,13 @@ func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request)
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get last comments stream for %s", siteID)
sinceTs := time.Now()
if since := r.URL.Query().Get("since"); since != "" {
unixTS, e := strconv.ParseInt(since, 10, 64)
if e != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, e, "can't translate since parameter", rest.ErrDecode)
return
}
sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
sinceTs, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
return
}
if sinceTs.IsZero() {
sinceTs = time.Now()
}
fn := func() steamEventFn {
@@ -273,8 +274,8 @@ func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request)
}
}
if err := s.streamer.Activate(r.Context(), fn, w); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't stream", rest.ErrInternal)
if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
}
}
@@ -499,3 +500,15 @@ func (s *public) applyView(comments []store.Comment, view string) []store.Commen
}
return comments
}
func (s *public) parseSince(r *http.Request) (time.Time, error) {
sinceTs := time.Time{}
if since := r.URL.Query().Get("since"); since != "" {
unixTS, e := strconv.ParseInt(since, 10, 64)
if e != nil {
return time.Time{}, errors.Wrap(e, "can't translate since parameter")
}
sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
}
return sinceTs, nil
}
backend/app/store/engine/bolt.go
+4 -2
View File
@@ -164,7 +164,7 @@ func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
}
switch {
case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
case req.Locator.SiteID != "" && req.Locator.URL != "": // find post comments, i.e. for site and url
err = bdb.View(func(tx *bolt.Tx) error {
bucket, e := b.getPostBucket(tx, req.Locator.URL)
@@ -177,7 +177,9 @@ func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
if e = json.Unmarshal(v, &comment); e != nil {
return errors.Wrap(e, "failed to unmarshal")
}
comments = append(comments, comment)
if req.Since.IsZero() || comment.Timestamp.After(req.Since) {
comments = append(comments, comment)
}
return nil
})
})
backend/app/store/engine/bolt_test.go
+23
View File
@@ -166,6 +166,29 @@ func TestBoltDB_FindLastSince(t *testing.T) {
assert.Equal(t, 0, len(res))
}
func TestBoltDB_FindInPostSince(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "-time", Since: ts}
res, err := b.Find(req)
assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
res, err = b.Find(req)
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
res, err = b.Find(req)
assert.NoError(t, err)
assert.Equal(t, 0, len(res))
}
func TestBoltDB_FindForUser(t *testing.T) {
var b, teardown = prep(t)
defer teardown()
backend/app/store/service/service.go
+8 -3
View File
@@ -101,10 +101,15 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
return s.Engine.Create(comment)
}
// Find wraps engine's Find call and alter results if needed
// user used to filter results for self vs others
// Find wraps engine's Find call and alter results if needed. User used to alter comments
// in order to differentiate between user's comments vs others comments.
func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
req := engine.FindRequest{Locator: locator, Sort: sort}
return s.FindSince(locator, sort, user, time.Time{})
}
// FindSince wraps engine's Find call and alter results if needed. Returns comments after since tx
func (s *DataStore) FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error) {
req := engine.FindRequest{Locator: locator, Sort: sort, Since: since}
comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
backend/app/store/service/service_test.go
+17
View File
@@ -802,6 +802,23 @@ func TestService_Find(t *testing.T) {
assert.InDelta(t, 0, res[1].Controversy, 0.01)
}
func TestService_FindSince(t *testing.T) {
// two comments for https://radio-t.com, no reply
b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
res, err := b.FindSince(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{}, time.Time{})
require.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "id-1", res[0].ID)
res, err = b.FindSince(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{},
time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local))
require.NoError(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "id-2", res[0].ID)
}
func TestService_Info(t *testing.T) {
defer teardown(t)