From ce92f63215e6bce8b941fa4835a7df82e0ad9bb7 Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Fri, 3 Apr 2020 06:19:08 +0200 Subject: [PATCH] add remote image storage support to backend --- backend/app/cmd/server.go | 17 +++- backend/app/store/image/bolt_store_test.go | 1 + backend/app/store/image/fs_store_test.go | 1 + backend/app/store/image/remote_store.go | 58 +++++++++++++ backend/app/store/image/remote_store_test.go | 90 ++++++++++++++++++++ 5 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 backend/app/store/image/remote_store.go create mode 100644 backend/app/store/image/remote_store_test.go diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index bfd85fdc..de5599e9 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -129,7 +129,7 @@ type StoreGroup struct { // ImageGroup defines options group for store pictures type ImageGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"rpc" default:"fs"` // nolint FS struct { Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"` Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"` @@ -138,9 +138,10 @@ type ImageGroup struct { Bolt struct { File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"` } `group:"bolt" namespace:"bolt" env-namespace:"bolt"` - MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"` - ResizeWidth int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"` - ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"` + MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"` + ResizeWidth int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"` + ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"` + RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"` } // AvatarGroup defines options group for avatar params @@ -590,6 +591,14 @@ func (s *ServerCommand) makePicturesStore() (*image.Service, error) { Staging: s.Image.FS.Staging, Partitions: s.Image.FS.Partitions, }, imageServiceParams), nil + case "rpc": + return image.NewService(&image.RPC{ + Client: jrpc.Client{ + API: s.Image.RPC.API, + Client: http.Client{Timeout: s.Image.RPC.TimeOut}, + AuthUser: s.Image.RPC.AuthUser, + AuthPasswd: s.Image.RPC.AuthPassword, + }}, imageServiceParams), nil } return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type) } diff --git a/backend/app/store/image/bolt_store_test.go b/backend/app/store/image/bolt_store_test.go index e7bd7cd3..89748f6f 100644 --- a/backend/app/store/image/bolt_store_test.go +++ b/backend/app/store/image/bolt_store_test.go @@ -56,6 +56,7 @@ func TestBoltStore_LoadAfterSave(t *testing.T) { data, err := svc.Load(id) assert.NoError(t, err) assert.Equal(t, 1462, len(data)) + assert.Equal(t, gopherPNGBytes(), data) _, err = svc.Load("abcd") assert.Error(t, err) diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go index 4c24d23e..134fc98a 100644 --- a/backend/app/store/image/fs_store_test.go +++ b/backend/app/store/image/fs_store_test.go @@ -113,6 +113,7 @@ func TestFsStore_LoadAfterSave(t *testing.T) { data, err := svc.Load(id) assert.NoError(t, err) assert.Equal(t, 1462, len(data)) + assert.Equal(t, gopherPNGBytes(), data) _, err = svc.Load("abcd") assert.Error(t, err) } diff --git a/backend/app/store/image/remote_store.go b/backend/app/store/image/remote_store.go new file mode 100644 index 00000000..b374fdf1 --- /dev/null +++ b/backend/app/store/image/remote_store.go @@ -0,0 +1,58 @@ +package image + +import ( + "context" + "encoding/base64" + "encoding/json" + "io/ioutil" + "strings" + "time" + + "github.com/go-pkgz/jrpc" +) + +// RPC implements remote engine and delegates all Calls to remote http server +type RPC struct { + jrpc.Client +} + +func (r *RPC) Save(userID string, img []byte) (id string, err error) { + resp, err := r.Call("image.save", userID, img) + if err != nil { + return "", err + } + err = json.Unmarshal(*resp.Result, &id) + return id, err +} + +func (r *RPC) SaveWithID(id string, img []byte) (string, error) { + resp, err := r.Call("image.save_with_id", id, img) + if err != nil { + return "", err + } + var newID string + err = json.Unmarshal(*resp.Result, &newID) + return newID, err +} + +func (r *RPC) Load(id string) ([]byte, error) { + resp, err := r.Call("image.load", id) + if err != nil { + return nil, err + } + var rawImg string + if err = json.Unmarshal(*resp.Result, &rawImg); err != nil { + return nil, err + } + return ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg))) +} + +func (r *RPC) Commit(id string) error { + _, err := r.Call("image.commit", id) + return err +} + +func (r *RPC) Cleanup(_ context.Context, ttl time.Duration) error { + _, err := r.Call("image.cleanup", ttl) + return err +} diff --git a/backend/app/store/image/remote_store_test.go b/backend/app/store/image/remote_store_test.go new file mode 100644 index 00000000..e22d8752 --- /dev/null +++ b/backend/app/store/image/remote_store_test.go @@ -0,0 +1,90 @@ +package image + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-pkgz/jrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemote_Save(t *testing.T) { + ts := testServer(t, fmt.Sprintf(`{"method":"image.save","params":["admin","%s"],"id":1}`, gopher), + `{"result":"12345","id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Save("admin", gopherPNGBytes()) + assert.NoError(t, err) + assert.Equal(t, "12345", res) +} + +func TestRemote_SaveWithID(t *testing.T) { + ts := testServer(t, fmt.Sprintf(`{"method":"image.save_with_id","params":["54321","%s"],"id":1}`, gopher), + `{"result":"12345","id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.SaveWithID("54321", gopherPNGBytes()) + assert.NoError(t, err) + assert.Equal(t, "12345", res) +} + +func TestRemote_Load(t *testing.T) { + ts := testServer(t, `{"method":"image.load","params":"54321","id":1}`, + fmt.Sprintf(`{"result":"%v","id":1}`, gopher)) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Load("54321") + assert.NoError(t, err) + assert.Equal(t, gopherPNGBytes(), res) +} + +func TestRemote_Commit(t *testing.T) { + ts := testServer(t, `{"method":"image.commit","params":"gopher_id","id":1}`, `{"id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + err := c.Commit("gopher_id") + assert.NoError(t, err) +} + +func TestRemote_Cleanup(t *testing.T) { + ts := testServer(t, `{"method":"image.cleanup","params":60000000000,"id":1}`, `{"id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + err := c.Cleanup(context.TODO(), time.Minute) + assert.NoError(t, err) +} + +func testServer(t *testing.T, req, resp string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, req, string(body)) + _, _ = fmt.Fprint(w, resp) + })) +}