From 9957ab642b8e837b3fd25ceb20bff386ff1abebf Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 7 Jul 2020 19:51:48 +0300 Subject: [PATCH] Add Imported field to comment, do not wait image submission for such comments --- README.md | 9 +++++++-- backend/app/migrator/disqus.go | 1 + backend/app/migrator/native.go | 1 + backend/app/migrator/wordpress.go | 1 + backend/app/store/comment.go | 1 + backend/app/store/image/image.go | 23 ++++++++++++----------- backend/app/store/image/image_test.go | 25 +++++++++++++------------ backend/app/store/service/service.go | 11 ++++++++--- 8 files changed, 44 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 514d7775..4d78ab52 100644 --- a/README.md +++ b/README.md @@ -292,9 +292,9 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu 1. Register a new application [using the Azure portal](https://docs.microsoft.com/en-us/graph/auth-register-app-v2). 2. Under **"Authentication/Platform configurations/Web"** enter the correct url constructed as domain + `/auth/microsoft/callback`. i.e. `https://example.mysite.com/auth/microsoft/callback` -3. In "Overview" take note of the **Application (client) ID** +3. In "Overview" take note of the **Application (client) ID** 4. Choose the new project from the top right project dropdown (only if another project is selected) -5. Select "Certificates & secrets" and click on "+ New Client Secret". +5. Select "Certificates & secrets" and click on "+ New Client Secret". ##### Twitter Auth Provider @@ -324,6 +324,11 @@ Optionally, anonymous access can be turned on. In this case an extra `anonymous` - name should be at least 3 characters long - name has to start from the letter and contains letters, numbers, underscores and spaces only. +### Importing comments + +Remark supports importing comments from Disqus, WordPress or native backup format. +All imported comments has `Imported` field set to `true`. + #### Initial import from Disqus 1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details. diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index a9e1d8eb..f2f6faba 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -135,6 +135,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { Text: d.cleanText(comment.Message), Timestamp: comment.CreatedAt, ParentID: comment.Pid.Val, + Imported: true, } if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus c.User.ID = "disqus_" + c.User.Name diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go index b086a360..30287b8c 100644 --- a/backend/app/migrator/native.go +++ b/backend/app/migrator/native.go @@ -155,6 +155,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { for { comment := store.Comment{} err = dec.Decode(&comment) + comment.Imported = true if err == io.EOF { break } diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go index c2872e15..cbb2a661 100644 --- a/backend/app/migrator/wordpress.go +++ b/backend/app/migrator/wordpress.go @@ -139,6 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { Text: comment.Content, Timestamp: comment.Date.time, ParentID: comment.PID, + Imported: true, } commentsCh <- commentFormatter.Format(c) stats.inpComments++ diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go index 85b2ae6d..e162142c 100644 --- a/backend/app/store/comment.go +++ b/backend/app/store/comment.go @@ -26,6 +26,7 @@ type Comment struct { Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response Pin bool `json:"pin,omitempty" bson:"pin,omitempty"` Deleted bool `json:"delete,omitempty" bson:"delete"` + Imported bool `json:"imported,omitempty" bson:"imported"` PostTitle string `json:"title,omitempty" bson:"title"` } diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index e8962e1e..fb607724 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -101,8 +101,17 @@ func NewService(s Store, p ServiceParams) *Service { return &Service{ServiceParams: p, store: s} } +// SubmitAndCommit multiple ids immediately +func (s *Service) SubmitAndCommit(idsFn func() []string) { + for _, id := range idsFn() { + if err := s.store.Commit(id); err != nil { + log.Printf("[WARN] failed to commit image %s", id) + } + } +} + // Submit multiple ids via function for delayed commit -func (s *Service) Submit(idsFn func() []string, ts time.Time) { +func (s *Service) Submit(idsFn func() []string) { if idsFn == nil || s == nil { return } @@ -118,11 +127,7 @@ func (s *Service) Submit(idsFn func() []string, ts time.Time) { for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL { time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close) } - for _, id := range req.idsFn() { - if err := s.store.Commit(id); err != nil { - log.Printf("[WARN] failed to commit image %s", id) - } - } + s.SubmitAndCommit(req.idsFn) atomic.AddInt32(&s.submitCount, -1) } log.Printf("[INFO] image submitter terminated") @@ -131,11 +136,7 @@ func (s *Service) Submit(idsFn func() []string, ts time.Time) { atomic.AddInt32(&s.submitCount, 1) - now := time.Now() - if ts.IsZero() || ts.After(now) { - ts = now - } - s.submitCh <- submitReq{idsFn: idsFn, TS: ts} + s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()} } // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index da5dc93c..3cf6eec5 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -135,14 +135,15 @@ func TestService_Cleanup(t *testing.T) { func TestService_Submit(t *testing.T) { store := MockStore{} - store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) + store.On("Commit", mock.Anything, mock.Anything).Times(7).Return(nil) svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}} - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) - svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) - svc.Submit(nil, time.Now()) - store.AssertNumberOfCalls(t, "Commit", 0) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(func() []string { return []string{"id6", "id7"} }) + svc.Submit(nil) + store.AssertNumberOfCalls(t, "Commit", 2) time.Sleep(time.Millisecond * 150) - store.AssertNumberOfCalls(t, "Commit", 5) + store.AssertNumberOfCalls(t, "Commit", 7) svc.Close(context.TODO()) } @@ -150,9 +151,9 @@ func TestService_Close(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}} - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) - svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) - svc.Submit(nil, time.Now()) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(nil) svc.Close(context.TODO()) store.AssertNumberOfCalls(t, "Commit", 5) } @@ -161,10 +162,10 @@ func TestService_SubmitDelay(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond}) - svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }, time.Now()) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) time.Sleep(150 * time.Millisecond) // let first batch to pass TTL - svc.Submit(func() []string { return []string{"id4", "id5"} }, time.Now()) - svc.Submit(nil, time.Now()) + svc.Submit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(nil) store.AssertNumberOfCalls(t, "Commit", 3) svc.Close(context.TODO()) store.AssertNumberOfCalls(t, "Commit", 5) diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 7607736b..979214fe 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -225,8 +225,7 @@ func (s *DataStore) ResubmitStagingImages(sites []string) error { // submitImages initiated delayed commit of all images from the comment uploaded to remark42 func (s *DataStore) submitImages(comment store.Comment) { - - s.ImageService.Submit(func() []string { // get all ids from comment's text + idsFn := func() []string { // get all ids from comment's text // this can be called after last edit, we have to retrieve fresh comment cc, err := s.Engine.Get(engine.GetRequest{Locator: comment.Locator, CommentID: comment.ID}) if err != nil { @@ -242,7 +241,13 @@ func (s *DataStore) submitImages(comment store.Comment) { log.Printf("[DEBUG] image ids extracted from %s - %+v", comment.ID, imgIds) } return imgIds - }, comment.Timestamp) + } + + if comment.Imported { + s.ImageService.SubmitAndCommit(idsFn) + } else { + s.ImageService.Submit(idsFn) + } } // prepareNewComment sets new comment fields, hashing and sanitizing data