mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

introduce image.Service commitTTL and cleanupTTL properties

Browse Source 
This allows having separate values of TTL for Commit and Cleanup
and moving them apart in time, also clarifying their connection
to EditTime which was previously outside of the package level.
This commit is contained in:
Dmitry Verkhoturov
2020-04-23 00:54:30 +02:00
parent c7f73a6c84
commit 5ae6dcf652
6 changed files with 39 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/cmd/server.go
+6 -6
View File
@@ -338,7 +338,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
if err != nil {
return nil, errors.Wrap(err, "failed to make pictures store")
}
log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
log.Printf("[DEBUG] image service for url=%s, EditDuration=%v", imageService.ImageAPI, imageService.EditDuration)
dataService := &service.DataStore{
Engine: storeEngine,
@@ -580,11 +580,11 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
imageServiceParams := image.ServiceParams{
ImageAPI: s.RemarkURL + "/api/v1/picture/",
TTL: 5 * s.EditDuration, // add extra time to image TTL for staging
MaxSize: s.Image.MaxSize,
MaxHeight: s.Image.ResizeHeight,
MaxWidth: s.Image.ResizeWidth,
ImageAPI: s.RemarkURL + "/api/v1/picture/",
EditDuration: s.EditDuration,
MaxSize: s.Image.MaxSize,
MaxHeight: s.Image.ResizeHeight,
MaxWidth: s.Image.ResizeWidth,
}
switch s.Image.Type {
case "bolt":
backend/app/rest/api/rest_private_test.go
+2 -2
View File
@@ -908,8 +908,8 @@ func TestRest_CreateWithPictures(t *testing.T) {
Staging: "/tmp/remark42/images.staging",
Location: "/tmp/remark42/images",
}, image.ServiceParams{
TTL: 100 * time.Millisecond,
MaxSize: 2000,
EditDuration: 100 * time.Millisecond,
MaxSize: 2000,
})
svc.privRest.imageService = imageService
backend/app/rest/api/rest_test.go
+2 -2
View File
@@ -376,8 +376,8 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
Partitions: 100,
Staging: tmp + "/pics-remark42/staging",
}, image.ServiceParams{
TTL: 100 * time.Millisecond,
MaxSize: 10000,
EditDuration: 100 * time.Millisecond,
MaxSize: 10000,
}),
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
backend/app/store/image/image.go
+22 -12
View File
@@ -31,7 +31,7 @@ import (
// Service wraps Store with common functions needed for any store implementation
// It also provides async Submit with func param retrieving all submitting ids.
// Submitted ids committed (i.e. moved from staging to final) on TTL expiration.
// Submitted ids committed (i.e. moved from staging to final) on commitTTL expiration.
type Service struct {
ServiceParams
@@ -45,11 +45,17 @@ type Service struct {
// ServiceParams contains externally adjustable parameters of Service
type ServiceParams struct {
TTL time.Duration // for how long file allowed on staging
ImageAPI string // image api matching path
MaxSize int
MaxHeight int
MaxWidth int
EditDuration time.Duration // edit period for comments
ImageAPI string // image api matching path
MaxSize int
MaxHeight int
MaxWidth int
// duration of time after which images are checked and committed if still
// present in the submitted comment after it's EditDuration is expired
commitTTL time.Duration
// duration of time after which images are deleted from staging
cleanupTTL time.Duration
}
// StoreInfo contains image store meta information
@@ -82,6 +88,10 @@ type submitReq struct {
// NewService returns new Service instance
func NewService(s Store, p ServiceParams) *Service {
p.commitTTL = p.EditDuration * 15 / 10 // Commit call on every 1.5 * EditDuration
p.cleanupTTL = p.EditDuration * 25 / 10 // Cleanup call on every 2.5 * EditDuration
// In case Cleanup and Submit start at the same time (case of stale staging images check
// on the program start) these TTL values guarantee that Commit will happen before Cleanup.
return &Service{ServiceParams: p, store: s}
}
@@ -98,8 +108,8 @@ func (s *Service) Submit(idsFn func() []string) {
go func() {
defer s.wg.Done()
for req := range s.submitCh {
// wait for TTL expiration with emergency pass on term
for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL/2 { // commit on a half of TTL
// wait for commitTTL expiration with emergency pass on term
for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL {
time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
}
for _, id := range req.idsFn() {
@@ -139,17 +149,17 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error)
return ids, nil
}
// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
// Cleanup runs periodic cleanup with cleanupTTL. Blocking loop, should be called inside of goroutine by consumer
func (s *Service) Cleanup(ctx context.Context) {
log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.cleanupTTL)
for {
select {
case <-ctx.Done():
log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
return
case <-time.After(s.TTL / 2): // cleanup call on every 1/2 TTL
if err := s.store.Cleanup(ctx, s.TTL); err != nil {
case <-time.After(s.cleanupTTL):
if err := s.store.Cleanup(ctx, s.cleanupTTL); err != nil {
log.Printf("[WARN] failed to cleanup, %v", err)
}
}
backend/app/store/image/image_test.go
+4 -4
View File
@@ -96,7 +96,7 @@ func TestService_Cleanup(t *testing.T) {
store := MockStore{}
store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{TTL: 100 * time.Millisecond}}
svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
defer cancel()
svc.Cleanup(ctx)
@@ -106,7 +106,7 @@ func TestService_Cleanup(t *testing.T) {
func TestService_Submit(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Millisecond * 100}}
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}}
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
svc.Submit(func() []string { return []string{"id4", "id5"} })
svc.Submit(nil)
@@ -119,7 +119,7 @@ func TestService_Submit(t *testing.T) {
func TestService_Close(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Hour * 24}}
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
svc.Submit(func() []string { return []string{"id4", "id5"} })
svc.Submit(nil)
@@ -130,7 +130,7 @@ func TestService_Close(t *testing.T) {
func TestService_SubmitDelay(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Millisecond * 100}}
svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
svc.Submit(func() []string { return []string{"id4", "id5"} })
backend/app/store/service/service_test.go
+3 -2
View File
@@ -1278,8 +1278,9 @@ func TestService_submitImages(t *testing.T) {
lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
mockStore := image.MockStore{}
mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil)
imgSvc := image.NewService(&mockStore, image.ServiceParams{TTL: 50 * time.Millisecond * 50})
mockStore.On("Commit", mock.Anything).Times(2).Return(nil)
imgSvc := image.NewService(&mockStore, image.ServiceParams{EditDuration: 50 * time.Millisecond})
defer imgSvc.Close(context.TODO())
// two comments for https://radio-t.com
eng, teardown := prepStoreEngine(t)