mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify image.Store interface

Browse Source
This commit is contained in:
Dmitry Verkhoturov
2020-04-19 16:21:09 -05:00
committed by Umputun
parent b5e31f3081
commit 36cee9cc66
16 changed files with 83 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/_example/memory_store/accessor/image.go
+2 -14
View File
@@ -8,13 +8,11 @@ package accessor
import (
"context"
"path"
"sync"
"time"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/rs/xid"
)
// MemImage implements image.Store with memory backend
@@ -35,18 +33,13 @@ func NewMemImageStore() *MemImage {
}
}
func (m *MemImage) Save(userID string, img []byte) (id string, err error) {
id = path.Join(userID, guid())
return m.SaveWithID(id, img)
}
func (m *MemImage) SaveWithID(id string, img []byte) (string, error) {
func (m *MemImage) SaveWithID(id string, img []byte) error {
m.Lock()
m.imagesStaging[id] = img
m.insertTime[id] = time.Now()
m.Unlock()
return id, nil
return nil
}
func (m *MemImage) Load(id string) ([]byte, error) {
@@ -98,8 +91,3 @@ func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
m.Unlock()
return nil
}
// guid makes a globally unique id
func guid() string {
return xid.New().String()
}
backend/_example/memory_store/accessor/image_test.go
+2 -15
View File
@@ -41,20 +41,6 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
func TestMemImage_Save(t *testing.T) {
svc := NewMemImageStore()
id, err := svc.Save("user1", []byte(gopher))
assert.NoError(t, err)
assert.Contains(t, id, "user1/")
}
func TestMemImage_SaveWithIDFail(t *testing.T) {
svc := NewMemImageStore()
id, err := svc.SaveWithID("test_id", []byte(gopher))
assert.NoError(t, err)
assert.Equal(t, id, "test_id")
}
func TestMemImage_LoadAfterSave(t *testing.T) {
svc := NewMemImageStore()
gopher, err := ioutil.ReadAll(gopherPNG())
@@ -64,7 +50,8 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
assert.EqualError(t, err, "image test_id not found")
assert.Empty(t, img)
id, err := svc.Save("user1", gopher)
id := "test_img"
err = svc.SaveWithID(id, gopher)
assert.NoError(t, err)
img, err = svc.Load(id)
backend/_example/memory_store/server/image.go
+2 -15
View File
@@ -15,19 +15,6 @@ import (
"github.com/go-pkgz/jrpc"
)
func (s *RPC) imgSaveHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
var req [2]string
if err := json.Unmarshal(params, &req); err != nil {
return jrpc.Response{Error: err.Error()}
}
img, err := base64.StdEncoding.DecodeString(req[1])
if err != nil {
return jrpc.Response{Error: err.Error()}
}
value, err := s.img.Save(req[0], img)
return jrpc.EncodeResponse(id, value, err)
}
func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
var req [2]string
if err := json.Unmarshal(params, &req); err != nil {
@@ -37,8 +24,8 @@ func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Resp
if err != nil {
return jrpc.Response{Error: err.Error()}
}
value, err := s.img.SaveWithID(req[0], img)
return jrpc.EncodeResponse(id, value, err)
err = s.img.SaveWithID(req[0], img)
return jrpc.EncodeResponse(id, nil, err)
}
func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
backend/_example/memory_store/server/image_test.go
+18 -31
View File
@@ -50,31 +50,6 @@ func gopherPNGBytes() []byte {
return img
}
func TestRPC_imgSaveHndl(t *testing.T) {
_, port, teardown := prepTestStore(t)
defer teardown()
api := fmt.Sprintf("http://localhost:%d/test", port)
ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
id, err := ri.Save("admin", gopherPNGBytes())
assert.NoError(t, err)
assert.Contains(t, id, "admin/", "id contains username")
err = ri.Commit(id)
assert.NoError(t, err)
}
func TestRPC_imgSaveWithIDHndl(t *testing.T) {
_, port, teardown := prepTestStore(t)
defer teardown()
api := fmt.Sprintf("http://localhost:%d/test", port)
ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
id, err := ri.SaveWithID("test_id", gopherPNGBytes())
assert.NoError(t, err)
assert.Equal(t, id, "test_id")
}
func TestRPC_imgLoadHndl(t *testing.T) {
_, port, teardown := prepTestStore(t)
defer teardown()
@@ -82,7 +57,8 @@ func TestRPC_imgLoadHndl(t *testing.T) {
ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
// save
id, err := ri.Save("admin", gopherPNGBytes())
id := "test_img"
err := ri.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
// load
@@ -100,6 +76,16 @@ func TestRPC_imgLoadHndl(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1462, len(img))
assert.Equal(t, gopherPNGBytes(), img)
// cleanup
err = ri.Cleanup(nil, time.Second)
assert.NoError(t, err)
// load after cleanup
img, err = ri.Load(id)
assert.NoError(t, err)
assert.Equal(t, 1462, len(img))
assert.Equal(t, gopherPNGBytes(), img)
}
func TestRPC_imgCommitHndlFail(t *testing.T) {
@@ -120,12 +106,15 @@ func TestRPC_imgCleanupHndl(t *testing.T) {
ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
// save
id, err := ri.Save("admin", gopherPNGBytes())
id := "test_img"
err := ri.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
// load
_, err = ri.Load(id)
img, err := ri.Load(id)
assert.NoError(t, err)
assert.Equal(t, 1462, len(img))
assert.Equal(t, gopherPNGBytes(), img)
// cleanup
err = ri.Cleanup(context.TODO(), time.Nanosecond)
@@ -133,7 +122,5 @@ func TestRPC_imgCleanupHndl(t *testing.T) {
// load after cleanup should fail
_, err = ri.Load(id)
assert.Error(t, err)
assert.Contains(t, err.Error(), "image admin/")
assert.Contains(t, err.Error(), "not found")
assert.EqualError(t, err, "image test_img not found")
}
backend/_example/memory_store/server/rpc.go
-1
View File
@@ -57,7 +57,6 @@ func (s *RPC) addHandlers() {
// image store handlers
s.Group("image", jrpc.HandlersGroup{
"save": s.imgSaveHndl,
"save_with_id": s.imgSaveWithIDHndl,
"load": s.imgLoadHndl,
"commit": s.imgCommitHndl,
backend/app/rest/proxy/image.go
+2 -2
View File
@@ -130,11 +130,11 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
// cache image from provided Reader using given ID
func (p Image) cacheImage(r io.Reader, imgID string) {
id, err := p.ImageService.SaveWithID(imgID, r)
err := p.ImageService.SaveWithID(imgID, r)
if err != nil {
log.Printf("[WARN] unable to save image to the storage: %+v", err)
}
p.ImageService.Submit(func() []string { return []string{id} })
p.ImageService.Submit(func() []string { return []string{imgID} })
}
// download an image.
backend/app/rest/proxy/image_test.go
+1 -1
View File
@@ -175,7 +175,7 @@ func TestImage_RoutesCachingImage(t *testing.T) {
encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Return("", nil)
imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Return(nil)
imageStore.On("Commit", mock.Anything).Once().Return(nil)
resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
backend/app/store/image/bolt_store.go
+3 -10
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/binary"
"path"
"time"
log "github.com/go-pkgz/lgr"
@@ -52,8 +51,8 @@ func NewBoltStorage(fileName string, options bolt.Options) (*Bolt, error) {
}, nil
}
// SaveWithID saves data from a reader, for given id
func (b *Bolt) SaveWithID(id string, img []byte) (string, error) {
// SaveWithID saves image for given id to staging bucket in DB
func (b *Bolt) SaveWithID(id string, img []byte) error {
err := b.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), img); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", id)
@@ -68,13 +67,7 @@ func (b *Bolt) SaveWithID(id string, img []byte) (string, error) {
return nil
})
return id, err
}
// Save data from reader to staging bucket in DB
func (b *Bolt) Save(userID string, img []byte) (id string, err error) {
id = path.Join(userID, guid())
return b.SaveWithID(id, img)
return err
}
// Commit file stored in staging bucket by copying it to permanent bucket
backend/app/store/image/bolt_store_test.go
+12 -13
View File
@@ -18,10 +18,10 @@ func TestBoltStore_SaveCommit(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
assert.Contains(t, id, "user1")
t.Log(id)
err = svc.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
@@ -48,10 +48,9 @@ func TestBoltStore_LoadAfterSave(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
assert.Contains(t, id, "user1")
t.Log(id)
data, err := svc.Load(id)
assert.NoError(t, err)
@@ -66,25 +65,25 @@ func TestBoltStore_Cleanup(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
save := func(file string, user string) (id string) {
id, err := svc.Save(user, gopherPNGBytes())
save := func(file string) (id string) {
err := svc.SaveWithID(file, gopherPNGBytes())
require.NoError(t, err)
checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error {
checkBoltImgData(t, svc.db, imagesStagedBktName, file, func(data []byte) error {
require.NotNil(t, data)
assert.Equal(t, 1462, len(data))
return nil
})
return id
return file
}
// save 3 images to staging
img1 := save("blah_ff1.png", "user1")
img1 := save("blah_ff1.png")
img1ts := time.Now()
time.Sleep(100 * time.Millisecond)
img2 := save("blah_ff2.png", "user1")
img2 := save("blah_ff2.png")
time.Sleep(100 * time.Millisecond)
img3 := save("blah_ff3.png", "user2")
img3 := save("blah_ff3.png")
err := svc.Cleanup(context.Background(), time.Since(img1ts)) // clean first images
assert.NoError(t, err)
backend/app/store/image/fs_store.go
+6 -12
View File
@@ -32,27 +32,21 @@ type FileSystem struct {
}
}
// SaveWithID saves data from a reader, with given id
func (f *FileSystem) SaveWithID(id string, img []byte) (string, error) {
// SaveWithID saves image with given id to local FS, staging directory.
// Files partitioned across multiple subdirectories, and the final path includes part, i.e. /location/user1/03/123-4567
func (f *FileSystem) SaveWithID(id string, img []byte) error {
dst := f.location(f.Staging, id)
if err := os.MkdirAll(path.Dir(dst), 0700); err != nil {
return "", errors.Wrap(err, "can't make image directory")
return errors.Wrap(err, "can't make image directory")
}
if err := ioutil.WriteFile(dst, img, 0600); err != nil {
return "", errors.Wrapf(err, "can't write image file with id %s", id)
return errors.Wrapf(err, "can't write image file with id %s", id)
}
log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(img))
return id, nil
}
// Save data from a reader to local FS, staging directory. Returns id as user/uuid
// Files partitioned across multiple subdirectories, and the final path includes part, i.e. /location/user1/03/123-4567
func (f *FileSystem) Save(userID string, img []byte) (id string, err error) {
tempId := path.Join(userID, guid()) // make id as user/uuid
return f.SaveWithID(tempId, img)
return nil
}
// Commit file stored in staging location by moving it to permanent location
backend/app/store/image/fs_store_test.go
+13 -12
View File
@@ -48,13 +48,11 @@ func TestFsStore_Save(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
assert.Contains(t, id, "user1/")
t.Log(id)
img := svc.location(svc.Staging, id)
t.Log(img)
data, err := ioutil.ReadFile(img)
assert.NoError(t, err)
assert.Equal(t, 1462, len(data))
@@ -69,10 +67,9 @@ func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
assert.NoError(t, err)
img, err := ioutil.ReadAll(fh)
assert.NoError(t, err)
id, err := svc.Save("user1", img)
id := "test_img"
err = svc.SaveWithID(id, img)
assert.NoError(t, err)
assert.Contains(t, id, "user1/")
t.Log(id)
imgPath := svc.location(svc.Staging, id)
t.Log(imgPath)
@@ -85,7 +82,8 @@ func TestFsStore_SaveAndCommit(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
require.NoError(t, err)
err = svc.Commit(id)
require.NoError(t, err)
@@ -106,7 +104,8 @@ func TestFsStore_LoadAfterSave(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
t.Log(id)
@@ -123,7 +122,8 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("user1", gopherPNGBytes())
id := "test_img"
err := svc.SaveWithID(id, gopherPNGBytes())
assert.NoError(t, err)
t.Log(id)
err = svc.Commit(id)
@@ -184,8 +184,9 @@ func TestFsStore_Cleanup(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
save := func(file string, user string) (path string) {
id, err := svc.Save(user, gopherPNGBytes())
save := func(file string, user string) (filePath string) {
id := path.Join(user, file)
err := svc.SaveWithID(id, gopherPNGBytes())
require.NoError(t, err)
img := svc.location(svc.Staging, id)
data, err := ioutil.ReadFile(img)
backend/app/store/image/image.go
+9 -13
View File
@@ -15,6 +15,7 @@ import (
"io"
"io/ioutil"
"net/http"
"path"
"strings"
"sync"
"sync/atomic"
@@ -58,9 +59,8 @@ type ServiceParams struct {
// Two-stage commit scheme is used for not storing images which are uploaded but later never used in the comments,
// e.g. when somebody uploaded a picture but did not sent the comment.
type Store interface {
Save(userID string, img []byte) (id string, err error) // get name and reader and returns ID of stored (staging) image
SaveWithID(id string, img []byte) (string, error) // store image for passed id to staging
Load(id string) ([]byte, error) // load image by ID. Caller has to close the reader.
SaveWithID(id string, img []byte) error // store image with passed id to staging
Load(id string) ([]byte, error) // load image by ID. Caller has to close the reader.
Commit(id string) error // move image from staging to permanent
Cleanup(ctx context.Context, ttl time.Duration) error // run removal loop for old images on staging
@@ -116,20 +116,19 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error)
if err != nil {
return nil, errors.Wrap(err, "can't create document")
}
result := []string{}
doc.Find("img").Each(func(i int, sl *goquery.Selection) {
if im, ok := sl.Attr("src"); ok {
if strings.Contains(im, s.ImageAPI) {
elems := strings.Split(im, "/")
if len(elems) >= 2 {
id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
result = append(result, id)
ids = append(ids, id)
}
}
}
})
return result, nil
return ids, nil
}
// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
@@ -183,18 +182,15 @@ func (s *Service) Load(id string) ([]byte, error) {
// Save wraps storage Save function, validating and resizing the image before calling it.
func (s *Service) Save(userID string, r io.Reader) (id string, err error) {
img, err := s.prepareImage(r)
if err != nil {
return "", err
}
return s.store.Save(userID, img)
id = path.Join(userID, guid())
return id, s.SaveWithID(id, r)
}
// SaveWithID wraps storage SaveWithID function, validating and resizing the image before calling it.
func (s *Service) SaveWithID(id string, r io.Reader) (string, error) {
func (s *Service) SaveWithID(id string, r io.Reader) error {
img, err := s.prepareImage(r)
if err != nil {
return "", err
return err
}
return s.store.SaveWithID(id, img)
}
backend/app/store/image/image_mock.go
+5 -33
View File
@@ -62,44 +62,16 @@ func (_m *MockStore) Load(id string) ([]byte, error) {
return r0, r1
}
// Save provides a mock function with given fields: userID, img
func (_m *MockStore) Save(userID string, img []byte) (string, error) {
ret := _m.Called(userID, img)
var r0 string
if rf, ok := ret.Get(0).(func(string, []byte) string); ok {
r0 = rf(userID, img)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, []byte) error); ok {
r1 = rf(userID, img)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SaveWithID provides a mock function with given fields: id, img
func (_m *MockStore) SaveWithID(id string, img []byte) (string, error) {
func (_m *MockStore) SaveWithID(id string, img []byte) error {
ret := _m.Called(id, img)
var r0 string
if rf, ok := ret.Get(0).(func(string, []byte) string); ok {
var r0 error
if rf, ok := ret.Get(0).(func(string, []byte) error); ok {
r0 = rf(id, img)
} else {
r0 = ret.Get(0).(string)
r0 = ret.Error(0)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, []byte) error); ok {
r1 = rf(id, img)
} else {
r1 = ret.Error(1)
}
return r0, r1
return r0
}
backend/app/store/image/image_test.go
+3 -10
View File
@@ -21,15 +21,9 @@ func TestService_SaveAndLoad(t *testing.T) {
store := MockStore{}
svc := NewService(&store, ServiceParams{MaxSize: 1500, MaxWidth: 32, MaxHeight: 32})
store.On("Save", "user1", mock.Anything).Return("user1/test_id", nil)
id, err := svc.Save("user1", gopherPNG())
store.On("SaveWithID", "test_id", mock.Anything).Return(nil)
err := svc.SaveWithID("test_id", gopherPNG())
assert.NoError(t, err)
assert.Equal(t, "user1/test_id", id)
store.On("SaveWithID", "test_id", mock.Anything).Return("test_id", nil)
id, err = svc.SaveWithID("test_id", gopherPNG())
assert.NoError(t, err)
assert.Equal(t, "test_id", id)
store.On("Load", "test_id", mock.Anything).Return(nil, nil)
img, err := svc.Load("test_id")
@@ -65,7 +59,7 @@ func TestService_SaveTooLarge(t *testing.T) {
_, err := svc.Save("user2", io.MultiReader(gopherPNG(), gopherPNG()))
assert.Error(t, err)
assert.Contains(t, err.Error(), "is too large")
_, err = svc.SaveWithID("test_id", io.MultiReader(gopherPNG(), gopherPNG()))
err = svc.SaveWithID("test_id", io.MultiReader(gopherPNG(), gopherPNG()))
assert.Error(t, err)
assert.Contains(t, err.Error(), "is too large")
}
@@ -147,7 +141,6 @@ func TestService_SubmitDelay(t *testing.T) {
}
func TestService_resize(t *testing.T) {
// reader is nil
resized := resize(nil, 100, 100)
assert.Nil(t, resized)
backend/app/store/image/remote_store.go
+3 -17
View File
@@ -16,23 +16,9 @@ type RPC struct {
jrpc.Client
}
func (r *RPC) Save(userID string, img []byte) (id string, err error) {
resp, err := r.Call("image.save", userID, img)
if err != nil {
return "", err
}
err = json.Unmarshal(*resp.Result, &id)
return id, err
}
func (r *RPC) SaveWithID(id string, img []byte) (string, error) {
resp, err := r.Call("image.save_with_id", id, img)
if err != nil {
return "", err
}
var newID string
err = json.Unmarshal(*resp.Result, &newID)
return newID, err
func (r *RPC) SaveWithID(id string, img []byte) error {
_, err := r.Call("image.save_with_id", id, img)
return err
}
func (r *RPC) Load(id string) ([]byte, error) {
backend/app/store/image/remote_store_test.go
+2 -17
View File
@@ -14,32 +14,17 @@ import (
"github.com/stretchr/testify/require"
)
func TestRemote_Save(t *testing.T) {
ts := testServer(t, fmt.Sprintf(`{"method":"image.save","params":["admin","%s"],"id":1}`, gopher),
`{"result":"12345","id":1}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
var a Store = &c
_ = a
res, err := c.Save("admin", gopherPNGBytes())
assert.NoError(t, err)
assert.Equal(t, "12345", res)
}
func TestRemote_SaveWithID(t *testing.T) {
ts := testServer(t, fmt.Sprintf(`{"method":"image.save_with_id","params":["54321","%s"],"id":1}`, gopher),
`{"result":"12345","id":1}`)
`{"id":1}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
var a Store = &c
_ = a
res, err := c.SaveWithID("54321", gopherPNGBytes())
err := c.SaveWithID("54321", gopherPNGBytes())
assert.NoError(t, err)
assert.Equal(t, "12345", res)
}
func TestRemote_Load(t *testing.T) {