add method to renew image cleanup timer

This commit is contained in:
Dmitry Verkhoturov
2021-05-24 17:33:59 -05:00
committed by Umputun
parent 82387729a4
commit 0e550e83fa
12 changed files with 114 additions and 9 deletions
@@ -45,6 +45,17 @@ func (m *MemImage) Save(id string, img []byte) error {
return nil
}
// ResetCleanupTimer resets cleanup timer for the image
func (m *MemImage) ResetCleanupTimer(id string) error {
m.Lock()
defer m.Unlock()
if _, ok := m.insertTime[id]; ok {
m.insertTime[id] = time.Now()
return nil
}
return errors.Errorf("image %s not found", id)
}
// Load image by ID
func (m *MemImage) Load(id string) ([]byte, error) {
m.RLock()
@@ -28,6 +28,16 @@ func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Resp
return jrpc.EncodeResponse(id, nil, err)
}
func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
var fileID string
if err := json.Unmarshal(params, &fileID); err != nil {
return jrpc.Response{Error: err.Error()}
}
err := s.img.ResetCleanupTimer(fileID)
return jrpc.EncodeResponse(id, nil, err)
}
func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
var fileID string
if err := json.Unmarshal(params, &fileID); err != nil {
@@ -116,7 +116,21 @@ func TestRPC_imgCleanupHndl(t *testing.T) {
assert.Equal(t, 1462, len(img))
assert.Equal(t, gopherPNGBytes(), img)
// cleanup
// wait for image to expire
time.Sleep(time.Millisecond * 50)
// reset the time to cleanup
err = ri.ResetCleanupTimer(id)
assert.NoError(t, err)
// cleanup, should not affect the new image
err = ri.Cleanup(context.TODO(), time.Millisecond*45)
assert.NoError(t, err)
// load after cleanup should succeed
_, err = ri.Load(id)
assert.NoError(t, err, "image is still on staging because it's cleanup timer was reset")
// cleanup with short TTL, should remove the image from staging
err = ri.Cleanup(context.TODO(), time.Nanosecond)
assert.NoError(t, err)
+6 -5
View File
@@ -57,10 +57,11 @@ func (s *RPC) addHandlers() {
// image store handlers
s.Group("image", jrpc.HandlersGroup{
"save_with_id": s.imgSaveWithIDHndl,
"load": s.imgLoadHndl,
"commit": s.imgCommitHndl,
"cleanup": s.imgCleanupHndl,
"info": s.imgInfoHndl,
"save_with_id": s.imgSaveWithIDHndl,
"reset_cleanup_timer": s.imgResetClnTimerHndl,
"load": s.imgLoadHndl,
"commit": s.imgCommitHndl,
"cleanup": s.imgCleanupHndl,
"info": s.imgInfoHndl,
})
}
+14
View File
@@ -81,6 +81,20 @@ func (b *Bolt) Commit(id string) error {
})
}
// ResetCleanupTimer resets cleanup timer for the image
func (b *Bolt) ResetCleanupTimer(id string) error {
return b.db.Update(func(tx *bolt.Tx) error {
tsBuf := &bytes.Buffer{}
if err := binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil {
return errors.Wrapf(err, "can't serialize timestamp for %s", id)
}
if err := tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", id)
}
return nil
})
}
// Load image from DB
func (b *Bolt) Load(id string) ([]byte, error) {
var data []byte
+4 -1
View File
@@ -95,10 +95,13 @@ func TestBoltStore_Cleanup(t *testing.T) {
err = svc.Commit(img3)
require.NoError(t, err)
// reset the time to cleanup
err = svc.ResetCleanupTimer(img2)
require.NoError(t, err)
err = svc.Cleanup(context.Background(), time.Millisecond*100)
assert.NoError(t, err)
assertBoltImgNil(t, svc.db, imagesStagedBktName, img2)
assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img2)
assertBoltImgNil(t, svc.db, imagesBktName, img2)
assertBoltImgNotNil(t, svc.db, imagesBktName, img3)
assert.NoError(t, err)
+14
View File
@@ -62,6 +62,20 @@ func (f *FileSystem) Commit(id string) error {
return errors.Wrapf(err, "failed to commit image %s", id)
}
// ResetCleanupTimer resets cleanup timer for the image
func (f *FileSystem) ResetCleanupTimer(id string) error {
file := f.location(f.Staging, id)
_, err := os.Stat(file)
if err != nil {
return errors.Wrapf(err, "can't get image stats for %s", id)
}
// we don't need to update access time (second arg),
// but reading it is platform-dependent and looks different on darwin and linux,
// so it's easier to update it as well
err = os.Chtimes(file, time.Now(), time.Now())
return errors.Wrapf(err, "problem updating %s modification time", file)
}
// Load image from FS. Uses id to get partition subdirectory.
func (f *FileSystem) Load(id string) ([]byte, error) {
+7 -2
View File
@@ -221,14 +221,19 @@ func TestFsStore_Cleanup(t *testing.T) {
_, err = os.Stat(img3)
assert.NoError(t, err, "file on staging")
time.Sleep(200 * time.Millisecond) // make all images expired
time.Sleep(200 * time.Millisecond) // make all images expired
err = svc.ResetCleanupTimer("user2/blah_ff3.png") // reset the time to cleanup for third image
assert.NoError(t, err)
err = svc.Cleanup(context.Background(), time.Millisecond*300)
assert.NoError(t, err)
_, err = os.Stat(img2)
assert.Error(t, err, "no file on staging anymore")
_, err = os.Stat(img3)
assert.Error(t, err, "no file on staging anymore")
assert.NoError(t, err, "third image is still on staging because it's cleanup timer was reset")
err = svc.ResetCleanupTimer("unknown_image.png")
assert.Error(t, err)
}
func TestFsStore_Info(t *testing.T) {
+1
View File
@@ -75,6 +75,7 @@ type Store interface {
Save(id string, img []byte) error // store image with passed id to staging
Load(id string) ([]byte, error) // load image by ID
ResetCleanupTimer(id string) error // resets cleanup timer for the image, called on comment preview
Commit(id string) error // move image from staging to permanent
Cleanup(ctx context.Context, ttl time.Duration) error // run removal loop for old images on staging
}
+14
View File
@@ -86,6 +86,20 @@ func (_m *MockStore) Load(id string) ([]byte, error) {
return r0, r1
}
// ResetCleanupTimer provides a mock function with given fields: id
func (_m *MockStore) ResetCleanupTimer(id string) error {
ret := _m.Called(id)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Save provides a mock function with given fields: id, img
func (_m *MockStore) Save(id string, img []byte) error {
ret := _m.Called(id, img)
+6
View File
@@ -22,6 +22,12 @@ func (r *RPC) Save(id string, img []byte) error {
return err
}
// ResetCleanupTimer resets cleanup timer for the image
func (r *RPC) ResetCleanupTimer(id string) error {
_, err := r.Call("image.reset_cleanup_timer", id)
return err
}
// Load image with given id
func (r *RPC) Load(id string) ([]byte, error) {
resp, err := r.Call("image.load", id)
@@ -53,6 +53,18 @@ func TestRemote_Commit(t *testing.T) {
assert.NoError(t, err)
}
func TestRemote_ResetCleanupTimer(t *testing.T) {
ts := testServer(t, `{"method":"image.reset_cleanup_timer","params":"gopher_id","id":1}`, `{"id":1}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
var a Store = &c
_ = a
err := c.ResetCleanupTimer("gopher_id")
assert.NoError(t, err)
}
func TestRemote_Cleanup(t *testing.T) {
ts := testServer(t, `{"method":"image.cleanup","params":60000000000,"id":1}`, `{"id":1}`)
defer ts.Close()