make image ids extraction safe for pos-edits, delays comment parsing

backend/app/store/image/fs_store.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"hash/crc64"
 	"io"
-	"log"
 	"math"
 	"os"
 	"path"
@@ -15,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )

backend/app/store/image/image.go

@@ -8,13 +8,13 @@ package image
 import (
 	"context"
 	"io"
-	"log"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 
@@ -42,17 +42,18 @@ type Service struct {
 const submitQueueSize = 5000
 
 type submitReq struct {
-	ID string
-	TS time.Time
+	idsFn func() (ids []string)
+	TS    time.Time
 }
 
-// Submit multiple ids for delayed commit
-func (s *Service) Submit(ids []string) {
-	if len(ids) == 0 {
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil {
 		return
 	}
 
 	s.once.Do(func() {
+		log.Printf("[DEBUG] image submiter activate")
 		s.submitCh = make(chan submitReq, submitQueueSize)
 		s.wg.Add(1)
 		go func() {
@@ -62,17 +63,17 @@ func (s *Service) Submit(ids []string) {
 				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
-				if err := s.Commit(req.ID); err != nil {
-					log.Printf("[WARN] failed to commit image %s", req.ID)
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
 				}
 			}
 			log.Printf("[INFO] image submiter terminated")
 		}()
 	})
 
-	for _, id := range ids {
-		s.submitCh <- submitReq{ID: id, TS: time.Now()}
-	}
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
 }
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png

backend/app/store/image/image_test.go

@@ -41,8 +41,8 @@ func TestService_Submit(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
-	svc.Submit([]string{"id1", "id2", "id3"})
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	time.Sleep(time.Millisecond * 500)
 }
@@ -54,8 +54,8 @@ func TestService_Close(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
-	svc.Submit([]string{"id1", "id2", "id3"})
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	svc.Close()
 }
@@ -70,8 +70,8 @@ func TestService_SubmitDelay(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
-	svc.Submit([]string{"id1", "id2", "id3"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 }

backend/app/store/service/service.go

@@ -92,11 +92,23 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
-	imgIds, err := s.ImageService.ExtractPictures(comment.Text)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to prepare extract pictures")
-	}
-	s.ImageService.Submit(imgIds) // submit images commit, delayed by EditDuration
+	// submit comment images to delayed processing
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, e := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if e != nil {
+			return nil
+		}
+		imgIds, e := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+
 	return s.Interface.Create(comment)
 }