From 8aa24341c35f057c18b8f0d2668346ca52685053 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 23 Mar 2019 23:30:53 -0500 Subject: [PATCH] make image ids extraction safe for pos-edits, delays comment parsing --- backend/app/store/image/fs_store.go | 2 +- backend/app/store/image/image.go | 23 ++++++++++++----------- backend/app/store/image/image_test.go | 12 ++++++------ backend/app/store/service/service.go | 22 +++++++++++++++++----- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go index ae6c4c64..b65de287 100644 --- a/backend/app/store/image/fs_store.go +++ b/backend/app/store/image/fs_store.go @@ -5,7 +5,6 @@ import ( "fmt" "hash/crc64" "io" - "log" "math" "os" "path" @@ -15,6 +14,7 @@ import ( "sync" "time" + log "github.com/go-pkgz/lgr" "github.com/google/uuid" "github.com/pkg/errors" ) diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index 8c5b620f..5f370014 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -8,13 +8,13 @@ package image import ( "context" "io" - "log" "strings" "sync" "sync/atomic" "time" "github.com/PuerkitoBio/goquery" + log "github.com/go-pkgz/lgr" "github.com/pkg/errors" ) @@ -42,17 +42,18 @@ type Service struct { const submitQueueSize = 5000 type submitReq struct { - ID string - TS time.Time + idsFn func() (ids []string) + TS time.Time } -// Submit multiple ids for delayed commit -func (s *Service) Submit(ids []string) { - if len(ids) == 0 { +// Submit multiple ids via function for delayed commit +func (s *Service) Submit(idsFn func() []string) { + if idsFn == nil { return } s.once.Do(func() { + log.Printf("[DEBUG] image submiter activate") s.submitCh = make(chan submitReq, submitQueueSize) s.wg.Add(1) go func() { @@ -62,17 +63,17 @@ func (s *Service) Submit(ids []string) { for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL { time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close) } - if err := s.Commit(req.ID); err != nil { - log.Printf("[WARN] failed to commit image %s", req.ID) + for _, id := range req.idsFn() { + if err := s.Commit(id); err != nil { + log.Printf("[WARN] failed to commit image %s", id) + } } } log.Printf("[INFO] image submiter terminated") }() }) - for _, id := range ids { - s.submitCh <- submitReq{ID: id, TS: time.Now()} - } + s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()} } // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 335327fa..f26eec8a 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -41,8 +41,8 @@ func TestService_Submit(t *testing.T) { store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} - svc.Submit([]string{"id1", "id2", "id3"}) - svc.Submit([]string{"id4", "id5"}) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id4", "id5"} }) svc.Submit(nil) time.Sleep(time.Millisecond * 500) } @@ -54,8 +54,8 @@ func TestService_Close(t *testing.T) { store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500} - svc.Submit([]string{"id1", "id2", "id3"}) - svc.Submit([]string{"id4", "id5"}) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id4", "id5"} }) svc.Submit(nil) svc.Close() } @@ -70,8 +70,8 @@ func TestService_SubmitDelay(t *testing.T) { store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} - svc.Submit([]string{"id1", "id2", "id3"}) + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) time.Sleep(150 * time.Millisecond) // let first batch to pass TTL - svc.Submit([]string{"id4", "id5"}) + svc.Submit(func() []string { return []string{"id4", "id5"} }) svc.Submit(nil) } diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index d65f5cbc..6f9634ac 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -92,11 +92,23 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) comment.PostTitle = title }() - imgIds, err := s.ImageService.ExtractPictures(comment.Text) - if err != nil { - return "", errors.Wrap(err, "failed to prepare extract pictures") - } - s.ImageService.Submit(imgIds) // submit images commit, delayed by EditDuration + // submit comment images to delayed processing + s.ImageService.Submit(func() []string { + c := comment + cc, e := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment + if e != nil { + return nil + } + imgIds, e := s.ImageService.ExtractPictures(cc.Text) + if err != nil { + return nil + } + if len(imgIds) > 0 { + log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds) + } + return imgIds + }) + return s.Interface.Create(comment) }