From 1f9566bb47c6b8bc3e9fc7e73554d84a610c602e Mon Sep 17 00:00:00 2001 From: vdimir Date: Sun, 5 Jul 2020 20:43:12 +0300 Subject: [PATCH] Use comment timestamp for submitting image ttl --- backend/app/store/image/image.go | 9 +++++++-- backend/app/store/image/image_test.go | 18 +++++++++--------- backend/app/store/service/service.go | 16 ++++++++-------- backend/app/store/service/service_test.go | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index 11566191..e8962e1e 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -102,7 +102,7 @@ func NewService(s Store, p ServiceParams) *Service { } // Submit multiple ids via function for delayed commit -func (s *Service) Submit(idsFn func() []string) { +func (s *Service) Submit(idsFn func() []string, ts time.Time) { if idsFn == nil || s == nil { return } @@ -130,7 +130,12 @@ func (s *Service) Submit(idsFn func() []string) { }) atomic.AddInt32(&s.submitCount, 1) - s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()} + + now := time.Now() + if ts.IsZero() || ts.After(now) { + ts = now + } + s.submitCh <- submitReq{idsFn: idsFn, TS: ts} } // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 19863807..da5dc93c 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -137,9 +137,9 @@ func TestService_Submit(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}} - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) - svc.Submit(func() []string { return []string{"id4", "id5"} }) - svc.Submit(nil) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) + svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) + svc.Submit(nil, time.Now()) store.AssertNumberOfCalls(t, "Commit", 0) time.Sleep(time.Millisecond * 150) store.AssertNumberOfCalls(t, "Commit", 5) @@ -150,9 +150,9 @@ func TestService_Close(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}} - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) - svc.Submit(func() []string { return []string{"id4", "id5"} }) - svc.Submit(nil) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) + svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) + svc.Submit(nil, time.Now()) svc.Close(context.TODO()) store.AssertNumberOfCalls(t, "Commit", 5) } @@ -161,10 +161,10 @@ func TestService_SubmitDelay(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond}) - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) time.Sleep(150 * time.Millisecond) // let first batch to pass TTL - svc.Submit(func() []string { return []string{"id4", "id5"} }) - svc.Submit(nil) + svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) + svc.Submit(nil, time.Now()) store.AssertNumberOfCalls(t, "Commit", 3) svc.Close(context.TODO()) store.AssertNumberOfCalls(t, "Commit", 5) diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 495b10b7..7607736b 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -101,7 +101,7 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) comment.PostTitle = title }() - s.submitImages(comment.Locator, comment.ID) + s.submitImages(comment) if e := s.AdminStore.OnEvent(comment.Locator.SiteID, admin.EvCreate); e != nil { log.Printf("[WARN] failed to send create event, %s", e) } @@ -217,32 +217,32 @@ func (s *DataStore) ResubmitStagingImages(sites []string) error { comments, err := s.FindSince(locator, "time", store.User{}, ts) result = multierror.Append(result, errors.Wrapf(err, "problem finding comments for site %s", site)) for _, c := range comments { - s.submitImages(c.Locator, c.ID) + s.submitImages(c) } } return result.ErrorOrNil() } // submitImages initiated delayed commit of all images from the comment uploaded to remark42 -func (s *DataStore) submitImages(locator store.Locator, commentID string) { +func (s *DataStore) submitImages(comment store.Comment) { s.ImageService.Submit(func() []string { // get all ids from comment's text // this can be called after last edit, we have to retrieve fresh comment - cc, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) + cc, err := s.Engine.Get(engine.GetRequest{Locator: comment.Locator, CommentID: comment.ID}) if err != nil { - log.Printf("[WARN] can't get comment's %s text for image extraction, %v", commentID, err) + log.Printf("[WARN] can't get comment's %s text for image extraction, %v", comment.ID, err) return nil } imgIds, err := s.ImageService.ExtractPictures(cc.Text) if err != nil { - log.Printf("[WARN] can't get extract pictures from %s, %v", commentID, err) + log.Printf("[WARN] can't get extract pictures from %s, %v", comment.ID, err) return nil } if len(imgIds) > 0 { - log.Printf("[DEBUG] image ids extracted from %s - %+v", commentID, imgIds) + log.Printf("[DEBUG] image ids extracted from %s - %+v", comment.ID, imgIds) } return imgIds - }) + }, comment.Timestamp) } // prepareNewComment sets new comment fields, hashing and sanitizing data diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index dcbf9d75..894dc546 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -1336,7 +1336,7 @@ func TestService_submitImages(t *testing.T) { _, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages assert.NoError(t, err) - b.submitImages(c.Locator, c.ID) + b.submitImages(c) time.Sleep(250 * time.Millisecond) mockStore.AssertNumberOfCalls(t, "Commit", 2) }