From c7f73a6c84e52d8e19bcc065d32135aa7da95f2d Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Thu, 23 Apr 2020 00:52:36 +0200 Subject: [PATCH] add image.Store.Info() function --- .../_example/memory_store/accessor/image.go | 16 +++++++++++ .../memory_store/accessor/image_test.go | 20 ++++++++++++++ backend/_example/memory_store/server/image.go | 5 ++++ .../memory_store/server/image_test.go | 22 +++++++++++++++ backend/_example/memory_store/server/rpc.go | 1 + backend/app/store/image/bolt_store.go | 23 ++++++++++++++++ backend/app/store/image/bolt_store_test.go | 19 +++++++++++++ backend/app/store/image/fs_store.go | 27 +++++++++++++++++++ backend/app/store/image/fs_store_test.go | 21 +++++++++++++-- backend/app/store/image/image.go | 11 ++++++++ backend/app/store/image/image_mock.go | 21 +++++++++++++++ backend/app/store/image/image_test.go | 11 ++++++++ backend/app/store/image/remote_store.go | 13 +++++++++ backend/app/store/image/remote_store_test.go | 14 ++++++++++ 14 files changed, 222 insertions(+), 2 deletions(-) diff --git a/backend/_example/memory_store/accessor/image.go b/backend/_example/memory_store/accessor/image.go index 6f4b385c..e667ec9b 100644 --- a/backend/_example/memory_store/accessor/image.go +++ b/backend/_example/memory_store/accessor/image.go @@ -13,6 +13,8 @@ import ( log "github.com/go-pkgz/lgr" "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store/image" ) // MemImage implements image.Store with memory backend @@ -95,3 +97,17 @@ func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error { m.Unlock() return nil } + +// Info returns meta information about storage +func (m *MemImage) Info() (image.StoreInfo, error) { + var ts time.Time + m.RLock() + for _, t := range m.insertTime { + if ts.IsZero() || t.Before(ts) { + ts = t + } + } + m.RUnlock() + + return image.StoreInfo{FirstStagingImageTS: ts}, nil +} diff --git a/backend/_example/memory_store/accessor/image_test.go b/backend/_example/memory_store/accessor/image_test.go index bf496d0b..92679c63 100644 --- a/backend/_example/memory_store/accessor/image_test.go +++ b/backend/_example/memory_store/accessor/image_test.go @@ -80,3 +80,23 @@ func TestMemImage_Cleanup(t *testing.T) { err := svc.Cleanup(context.TODO(), time.Minute) assert.NoError(t, err) } + +func TestMemImage_Info(t *testing.T) { + svc := NewMemImageStore() + gopher, err := ioutil.ReadAll(gopherPNG()) + assert.NoError(t, err) + + // get info on empty storage, should be zero + info, err := svc.Info() + assert.NoError(t, err) + assert.True(t, info.FirstStagingImageTS.IsZero()) + + // save image + err = svc.Save("test_img", gopher) + assert.NoError(t, err) + + // get info after saving, should be non-zero + info, err = svc.Info() + assert.NoError(t, err) + assert.False(t, info.FirstStagingImageTS.IsZero()) +} diff --git a/backend/_example/memory_store/server/image.go b/backend/_example/memory_store/server/image.go index 9b550206..ab90bd9d 100644 --- a/backend/_example/memory_store/server/image.go +++ b/backend/_example/memory_store/server/image.go @@ -54,3 +54,8 @@ func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Respons err := s.img.Cleanup(context.TODO(), ttl) return jrpc.EncodeResponse(id, nil, err) } + +func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) { + info, err := s.img.Info() + return jrpc.EncodeResponse(id, info, err) +} diff --git a/backend/_example/memory_store/server/image_test.go b/backend/_example/memory_store/server/image_test.go index d264f3ec..0a5ea027 100644 --- a/backend/_example/memory_store/server/image_test.go +++ b/backend/_example/memory_store/server/image_test.go @@ -124,3 +124,25 @@ func TestRPC_imgCleanupHndl(t *testing.T) { _, err = ri.Load(id) assert.EqualError(t, err, "image test_img not found") } + +func TestRPC_imgInfoHndl(t *testing.T) { + port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + + // get info on empty storage, should be zero + info, err := ri.Info() + assert.NoError(t, err) + assert.True(t, info.FirstStagingImageTS.IsZero()) + + // save + err = ri.Save("test_img", gopherPNGBytes()) + assert.NoError(t, err) + + // get info after saving, should be non-zero + info, err = ri.Info() + assert.NoError(t, err) + assert.False(t, info.FirstStagingImageTS.IsZero()) +} diff --git a/backend/_example/memory_store/server/rpc.go b/backend/_example/memory_store/server/rpc.go index 8dbe6568..07609e7d 100644 --- a/backend/_example/memory_store/server/rpc.go +++ b/backend/_example/memory_store/server/rpc.go @@ -61,5 +61,6 @@ func (s *RPC) addHandlers() { "load": s.imgLoadHndl, "commit": s.imgCommitHndl, "cleanup": s.imgCleanupHndl, + "info": s.imgInfoHndl, }) } diff --git a/backend/app/store/image/bolt_store.go b/backend/app/store/image/bolt_store.go index 5ed8d68c..ff1ac801 100644 --- a/backend/app/store/image/bolt_store.go +++ b/backend/app/store/image/bolt_store.go @@ -140,3 +140,26 @@ func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error { }) return err } + +// Info returns meta information about storage +func (b *Bolt) Info() (StoreInfo, error) { + var ts time.Time + err := b.db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(insertTimeBktName)).Cursor() + + for id, tsData := c.First(); id != nil; id, tsData = c.Next() { + var createdRaw int64 + err := binary.Read(bytes.NewReader(tsData), binary.LittleEndian, &createdRaw) + if err != nil { + return errors.Wrapf(err, "failed to deserialize timestamp for %s", id) + } + + created := time.Unix(0, createdRaw) + if ts.IsZero() || created.Before(ts) { + ts = created + } + } + return nil + }) + return StoreInfo{FirstStagingImageTS: ts}, errors.Wrapf(err, "problem retrieving first timestamp from staging images") +} diff --git a/backend/app/store/image/bolt_store_test.go b/backend/app/store/image/bolt_store_test.go index 2dce4cc5..14c7dc88 100644 --- a/backend/app/store/image/bolt_store_test.go +++ b/backend/app/store/image/bolt_store_test.go @@ -105,6 +105,25 @@ func TestBoltStore_Cleanup(t *testing.T) { assert.NoError(t, err) } +func TestBolt_Info(t *testing.T) { + svc, teardown := prepareBoltImageStorageTest(t) + defer teardown() + + // get info on empty storage, should be zero + info, err := svc.Info() + assert.NoError(t, err) + assert.True(t, info.FirstStagingImageTS.IsZero()) + + // save image + err = svc.Save("test_img", gopherPNGBytes()) + assert.NoError(t, err) + + // get info after saving, should be non-zero + info, err = svc.Info() + assert.NoError(t, err) + assert.False(t, info.FirstStagingImageTS.IsZero()) +} + func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) { checkBoltImgData(t, db, bucket, id, func(data []byte) error { assert.Nil(t, data, id) diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go index 420dde59..5306dc35 100644 --- a/backend/app/store/image/fs_store.go +++ b/backend/app/store/image/fs_store.go @@ -115,6 +115,33 @@ func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error { return errors.Wrap(err, "failed to cleanup images") } +// Info returns meta information about storage +func (f *FileSystem) Info() (StoreInfo, error) { + if _, err := os.Stat(f.Staging); os.IsNotExist(err) { + return StoreInfo{}, nil + } + + var ts time.Time + err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + created := info.ModTime() + if ts.IsZero() || created.Before(ts) { + ts = created + } + return nil + }) + if err != nil { + return StoreInfo{}, errors.Wrapf(err, "problem retrieving first timestamp from staging images on fs") + } + return StoreInfo{FirstStagingImageTS: ts}, nil +} + // location gets full path for id by adding partition to the final path in order to keep files in different subdirectories // and avoid too many files in a single place. // the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png. diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go index 17668cce..9ac57a44 100644 --- a/backend/app/store/image/fs_store_test.go +++ b/backend/app/store/image/fs_store_test.go @@ -107,7 +107,6 @@ func TestFsStore_LoadAfterSave(t *testing.T) { id := "test_img" err := svc.Save(id, gopherPNGBytes()) assert.NoError(t, err) - t.Log(id) data, err := svc.Load(id) assert.NoError(t, err) @@ -125,7 +124,6 @@ func TestFsStore_LoadAfterCommit(t *testing.T) { id := "test_img" err := svc.Save(id, gopherPNGBytes()) assert.NoError(t, err) - t.Log(id) err = svc.Commit(id) require.NoError(t, err) @@ -233,6 +231,25 @@ func TestFsStore_Cleanup(t *testing.T) { assert.Error(t, err, "no file on staging anymore") } +func TestFsStore_Info(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + + // get ts on empty storage, should be zero + ts, err := svc.Info() + assert.NoError(t, err) + assert.True(t, ts.FirstStagingImageTS.IsZero()) + + // save image + err = svc.Save("test_img", gopherPNGBytes()) + assert.NoError(t, err) + + // get ts after saving, should be non-zero + ts, err = svc.Info() + assert.NoError(t, err) + assert.False(t, ts.FirstStagingImageTS.IsZero()) +} + func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) { loc, err := ioutil.TempDir("", "test_image_r42") require.NoError(t, err, "failed to make temp dir") diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index e121aee2..e88eb997 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -52,6 +52,11 @@ type ServiceParams struct { MaxWidth int } +// StoreInfo contains image store meta information +type StoreInfo struct { + FirstStagingImageTS time.Time +} + // To regenerate mock run from this directory: // sh -c "mockery -inpkg -name Store -print > /tmp/image-mock.tmp && mv /tmp/image-mock.tmp image_mock.go" @@ -60,6 +65,7 @@ type ServiceParams struct { // Two-stage commit scheme is used for not storing images which are uploaded but later never used in the comments, // e.g. when somebody uploaded a picture but did not sent the comment. type Store interface { + Info() (StoreInfo, error) // get meta information about storage Save(id string, img []byte) error // store image with passed id to staging Load(id string) ([]byte, error) // load image by ID @@ -150,6 +156,11 @@ func (s *Service) Cleanup(ctx context.Context) { } } +// Info returns meta information about storage +func (s *Service) Info() (StoreInfo, error) { + return s.store.Info() +} + // Close flushes all in-progress submits and enforces waiting commits func (s *Service) Close(ctx context.Context) { log.Printf("[INFO] close image service ") diff --git a/backend/app/store/image/image_mock.go b/backend/app/store/image/image_mock.go index 25e6775d..f9c74ee4 100644 --- a/backend/app/store/image/image_mock.go +++ b/backend/app/store/image/image_mock.go @@ -39,6 +39,27 @@ func (_m *MockStore) Commit(id string) error { return r0 } +// Info provides a mock function with given fields: +func (_m *MockStore) Info() (StoreInfo, error) { + ret := _m.Called() + + var r0 StoreInfo + if rf, ok := ret.Get(0).(func() StoreInfo); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(StoreInfo) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Load provides a mock function with given fields: id func (_m *MockStore) Load(id string) ([]byte, error) { ret := _m.Called(id) diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 228ac45e..6a738f30 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -140,6 +140,17 @@ func TestService_SubmitDelay(t *testing.T) { store.AssertNumberOfCalls(t, "Commit", 5) } +func TestService_Info(t *testing.T) { + store := MockStore{} + store.On("Info", mock.Anything, mock.Anything).Once().Return(StoreInfo{}, nil) + + svc := Service{store: &store, ServiceParams: ServiceParams{}} + info, err := svc.Info() + assert.NoError(t, err) + assert.True(t, info.FirstStagingImageTS.IsZero()) + store.AssertNumberOfCalls(t, "Info", 1) +} + func TestService_resize(t *testing.T) { // reader is nil resized := resize(nil, 100, 100) diff --git a/backend/app/store/image/remote_store.go b/backend/app/store/image/remote_store.go index 13a60faa..b0048157 100644 --- a/backend/app/store/image/remote_store.go +++ b/backend/app/store/image/remote_store.go @@ -46,3 +46,16 @@ func (r *RPC) Cleanup(_ context.Context, ttl time.Duration) error { _, err := r.Call("image.cleanup", ttl) return err } + +// Info returns meta information about storage +func (r *RPC) Info() (StoreInfo, error) { + resp, err := r.Call("image.info") + if err != nil { + return StoreInfo{}, err + } + info := StoreInfo{} + if err = json.Unmarshal(*resp.Result, &info); err != nil { + return StoreInfo{}, err + } + return info, err +} diff --git a/backend/app/store/image/remote_store_test.go b/backend/app/store/image/remote_store_test.go index e0e214f2..77185b11 100644 --- a/backend/app/store/image/remote_store_test.go +++ b/backend/app/store/image/remote_store_test.go @@ -65,6 +65,20 @@ func TestRemote_Cleanup(t *testing.T) { assert.NoError(t, err) } +func TestRemote_Info(t *testing.T) { + ts := testServer(t, `{"method":"image.info","id":1}`, + `{"result":{"FirstStagingImageTS":"0001-01-01T00:00:01Z"},"id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + info, err := c.Info() + assert.NoError(t, err) + assert.False(t, info.FirstStagingImageTS.IsZero()) +} + func testServer(t *testing.T, req, resp string) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body)