From 09f06f587c814fa359c2a6d3dbd05465159c7775 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 8 Jul 2020 19:28:37 +0300 Subject: [PATCH] Add tests for comment.Imported, refine image.SubmitAndCommit tests --- backend/app/migrator/disqus_test.go | 2 ++ backend/app/migrator/native_test.go | 4 +++- backend/app/migrator/wordpress_test.go | 2 ++ backend/app/store/image/image.go | 16 ++++++++++++---- backend/app/store/image/image_test.go | 7 ++++--- backend/app/store/service/service.go | 13 +++++++++---- 6 files changed, 32 insertions(+), 12 deletions(-) diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 3eaaba05..47030848 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -39,6 +39,7 @@ func TestDisqus_Import(t *testing.T) { assert.Equal(t, "Alexander Blah", c.User.Name) assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID) assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP) + assert.True(t, c.Imported) posts, err := dataStore.List("test", 0, 0) assert.NoError(t, err) @@ -71,6 +72,7 @@ func TestDisqus_Convert(t *testing.T) { ID: "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", IP: "178.178.178.178", }, + Imported: true, } exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z") assert.Equal(t, exp0, res[0]) diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go index f30b05e7..10c8250e 100644 --- a/backend/app/migrator/native_test.go +++ b/backend/app/migrator/native_test.go @@ -73,7 +73,7 @@ func TestNative_Import(t *testing.T) { inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, link","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"} - {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + {"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00","imported":false}` b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "") r := Native{DataStore: b} @@ -87,10 +87,12 @@ func TestNative_Import(t *testing.T) { assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID) assert.Equal(t, "1234", comments[0].ParentID) assert.Equal(t, false, b.IsReadOnly(comments[0].Locator)) + assert.True(t, comments[0].Imported) assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID) assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL) assert.Equal(t, true, b.IsReadOnly(comments[1].Locator)) + assert.True(t, comments[1].Imported) assert.Equal(t, false, b.IsBlocked("radio-t", "user1")) assert.Equal(t, true, b.IsVerified("radio-t", "user1")) diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index 659842d6..79bb7226 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -42,6 +42,7 @@ func TestWordPress_Import(t *testing.T) { ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14") assert.Equal(t, ts, c.Timestamp) assert.Equal(t, c.Text, "

Mekkatorque was over in that tent up to the right

\n") + assert.True(t, c.Imported) posts, err := dataStore.List(siteID, 0, 0) assert.NoError(t, err) @@ -77,6 +78,7 @@ func TestWordPress_Convert(t *testing.T) { ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"), IP: "74.200.244.101", }, + Imported: true, } exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08") assert.Equal(t, exp1, comments[1]) diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index fb607724..c346d490 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -29,6 +29,7 @@ import ( "github.com/PuerkitoBio/goquery" log "github.com/go-pkgz/lgr" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/rs/xid" "golang.org/x/image/draw" @@ -102,12 +103,15 @@ func NewService(s Store, p ServiceParams) *Service { } // SubmitAndCommit multiple ids immediately -func (s *Service) SubmitAndCommit(idsFn func() []string) { +func (s *Service) SubmitAndCommit(idsFn func() []string) error { + errs := new(multierror.Error) for _, id := range idsFn() { - if err := s.store.Commit(id); err != nil { - log.Printf("[WARN] failed to commit image %s", id) + err := s.store.Commit(id) + if err != nil { + errs = multierror.Append(errs, errors.Wrapf(err, "failed to commit image %s", id)) } } + return errs.ErrorOrNil() } // Submit multiple ids via function for delayed commit @@ -127,7 +131,11 @@ func (s *Service) Submit(idsFn func() []string) { for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL { time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close) } - s.SubmitAndCommit(req.idsFn) + err := s.SubmitAndCommit(req.idsFn) + if err != nil { + log.Printf("[WARN] image commit error %v", err) + } + atomic.AddInt32(&s.submitCount, -1) } log.Printf("[INFO] image submitter terminated") diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 3cf6eec5..04e23790 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -136,13 +136,14 @@ func TestService_Cleanup(t *testing.T) { func TestService_Submit(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(7).Return(nil) - svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}} + svc := NewService(&store, ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}) svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) - svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} }) + err := svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} }) + assert.NoError(t, err) svc.Submit(func() []string { return []string{"id6", "id7"} }) svc.Submit(nil) store.AssertNumberOfCalls(t, "Commit", 2) - time.Sleep(time.Millisecond * 150) + time.Sleep(time.Millisecond * 175) store.AssertNumberOfCalls(t, "Commit", 7) svc.Close(context.TODO()) } diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index e03c616f..d1ee262b 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -101,13 +101,13 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) comment.PostTitle = title }() - defer s.submitImages(comment) // submit images only after creation in Engine + commentID, err = s.Engine.Create(comment) + s.submitImages(comment) if e := s.AdminStore.OnEvent(comment.Locator.SiteID, admin.EvCreate); e != nil { log.Printf("[WARN] failed to send create event, %s", e) } - - return s.Engine.Create(comment) + return commentID, err } // Find wraps engine's Find call and alter results if needed. User used to alter comments @@ -244,11 +244,16 @@ func (s *DataStore) submitImages(comment store.Comment) { return imgIds } + var err error if comment.Imported { - s.ImageService.SubmitAndCommit(idsFn) + err = s.ImageService.SubmitAndCommit(idsFn) } else { s.ImageService.Submit(idsFn) } + + if err != nil { + log.Printf("[WARN] failed to commit comment's images: %v", err) + } } // prepareNewComment sets new comment fields, hashing and sanitizing data