diff --git a/.github/workflows/ci-test-backend.yml b/.github/workflows/ci-test-backend.yml index cd2c10e7..b84be27f 100644 --- a/.github/workflows/ci-test-backend.yml +++ b/.github/workflows/ci-test-backend.yml @@ -29,7 +29,7 @@ jobs: - name: install go uses: actions/setup-go@v1 with: - go-version: 1.13 + go-version: 1.14 - name: test backend run: | diff --git a/backend/.golangci.yml b/backend/.golangci.yml index 0128643d..5e105464 100644 --- a/backend/.golangci.yml +++ b/backend/.golangci.yml @@ -21,4 +21,7 @@ linters: - varcheck disable-all: true issues: - exclude-use-default: false \ No newline at end of file + exclude-use-default: false + +service: + golangci-lint-version: 1.24.x \ No newline at end of file diff --git a/backend/_example/memory_store/README.md b/backend/_example/memory_store/README.md index 475703a4..c2904233 100644 --- a/backend/_example/memory_store/README.md +++ b/backend/_example/memory_store/README.md @@ -4,10 +4,10 @@ In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run: -1. docker-compose -f compose-dev-memstore.yml build -1. docker-compose -f compose-dev-memstore.yml up +1. `docker-compose -f compose-dev-memstore.yml build` +1. `docker-compose -f compose-dev-memstore.yml up` As usual, demo site will run on http://127.0.0.1:8080/web/ -note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package -. In real-life usage `replace github.com/umputun/remark/backend => ../../` should not be used. \ No newline at end of file +note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. +In real-life usage `replace github.com/umputun/remark/backend => ../../` should not be used. \ No newline at end of file diff --git a/backend/_example/memory_store/accessor/image.go b/backend/_example/memory_store/accessor/image.go new file mode 100644 index 00000000..03cf873c --- /dev/null +++ b/backend/_example/memory_store/accessor/image.go @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package accessor + +import ( + "context" + "path" + "sync" + "time" + + log "github.com/go-pkgz/lgr" + "github.com/pkg/errors" + "github.com/rs/xid" +) + +// MemImage implements image.Store with memory backend +type MemImage struct { + imagesStaging map[string][]byte + images map[string][]byte + insertTime map[string]time.Time + sync.RWMutex +} + +// NewMemImageStore makes admin Store in memory. +func NewMemImageStore() *MemImage { + log.Print("[DEBUG] make memory image store") + return &MemImage{ + imagesStaging: map[string][]byte{}, + images: map[string][]byte{}, + insertTime: map[string]time.Time{}, + } +} + +func (m *MemImage) Save(userID string, img []byte) (id string, err error) { + id = path.Join(userID, guid()) + return m.SaveWithID(id, img) +} + +func (m *MemImage) SaveWithID(id string, img []byte) (string, error) { + m.Lock() + m.imagesStaging[id] = img + m.insertTime[id] = time.Now() + m.Unlock() + + return id, nil +} + +func (m *MemImage) Load(id string) ([]byte, error) { + m.RLock() + img, ok := m.images[id] + if !ok { + img, ok = m.imagesStaging[id] + } + m.RUnlock() + if !ok { + return nil, errors.Errorf("image %s not found", id) + } + return img, nil +} + +func (m *MemImage) Commit(id string) error { + m.RLock() + img, ok := m.imagesStaging[id] + m.RUnlock() + if !ok { + return errors.Errorf("failed to commit %s, not found in staging", id) + } + + m.Lock() + m.images[id] = img + m.Unlock() + + return nil +} + +func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error { + var idsToRemove []string + + m.RLock() + for id, t := range m.insertTime { + age := time.Since(t) + if age > ttl { + log.Printf("[INFO] remove staging image %s, age %v", id, age) + idsToRemove = append(idsToRemove, id) + } + } + m.RUnlock() + + m.Lock() + for _, id := range idsToRemove { + delete(m.insertTime, id) + delete(m.imagesStaging, id) + } + m.Unlock() + return nil +} + +// guid makes a globally unique id +func guid() string { + return xid.New().String() +} diff --git a/backend/_example/memory_store/accessor/image_test.go b/backend/_example/memory_store/accessor/image_test.go new file mode 100644 index 00000000..3e74e7f4 --- /dev/null +++ b/backend/_example/memory_store/accessor/image_test.go @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package accessor + +import ( + "context" + "encoding/base64" + "io" + "io/ioutil" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// gopher png for test, from https://golang.org/src/image/png/example_test.go +const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" + + "+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" + + "73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" + + "/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" + + "JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" + + "8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" + + "ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" + + "MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" + + "MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" + + "rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" + + "rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" + + "dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" + + "yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" + + "LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" + + "11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" + + "A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" + + "yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" + + "1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" + + "nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==" + +func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } + +func TestMemImage_Save(t *testing.T) { + svc := NewMemImageStore() + id, err := svc.Save("user1", []byte(gopher)) + assert.NoError(t, err) + assert.Contains(t, id, "user1/") +} + +func TestMemImage_SaveWithIDFail(t *testing.T) { + svc := NewMemImageStore() + id, err := svc.SaveWithID("test_id", []byte(gopher)) + assert.NoError(t, err) + assert.Equal(t, id, "test_id") +} + +func TestMemImage_LoadAfterSave(t *testing.T) { + svc := NewMemImageStore() + gopher, err := ioutil.ReadAll(gopherPNG()) + assert.NoError(t, err) + + img, err := svc.Load("test_id") + assert.EqualError(t, err, "image test_id not found") + assert.Empty(t, img) + + id, err := svc.Save("user1", gopher) + assert.NoError(t, err) + + img, err = svc.Load(id) + assert.NoError(t, err) + assert.Equal(t, gopher, img) + + err = svc.Commit(id) + assert.NoError(t, err) + + err = svc.Cleanup(context.TODO(), 0) + assert.NoError(t, err) + + img, err = svc.Load(id) + assert.NoError(t, err) + assert.Equal(t, gopher, img) +} + +func TestMemImage_CommitFail(t *testing.T) { + svc := NewMemImageStore() + err := svc.Commit("test_id") + assert.EqualError(t, err, "failed to commit test_id, not found in staging") +} + +func TestMemImage_Cleanup(t *testing.T) { + svc := NewMemImageStore() + err := svc.Cleanup(context.TODO(), time.Minute) + assert.NoError(t, err) +} diff --git a/backend/_example/memory_store/compose-dev-memstore.yml b/backend/_example/memory_store/compose-dev-memstore.yml index 3392ecae..6d4d5045 100644 --- a/backend/_example/memory_store/compose-dev-memstore.yml +++ b/backend/_example/memory_store/compose-dev-memstore.yml @@ -7,16 +7,22 @@ services: remark42: build: - context: . + context: ../../.. dockerfile: Dockerfile args: - SKIP_BACKEND_TEST=true - SKIP_FRONTEND_TEST=true image: umputun/remark42:dev - container_name: "remark42" - hostname: "remark42" + container_name: "remark42-dev" + hostname: "remark42-dev" restart: always + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + ports: - "8080:8080" # primary rest server - "8084:8084" # local oauth2 server @@ -33,14 +39,26 @@ services: - ADMIN_RPC_API=http://mem_store.r42:8080/cmd - STORE_TYPE=rpc - STORE_RPC_API=http://mem_store.r42:8080/cmd + - IMAGE_TYPE=rpc + - IMAGE_RPC_API=http://mem_store.r42:8080/cmd + + volumes: + - ../../../var:/srv/var mem_store.r42: image: umputun/mem_store.r42 build: - context: . + context: ../../.. dockerfile: backend/_example/memory_store/Dockerfile container_name: "mem_store.r42" hostname: "mem_store.r42" + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + environment: - API=/cmd - SECRET=123456 diff --git a/backend/_example/memory_store/go.mod b/backend/_example/memory_store/go.mod index 45046f26..c2a5eca0 100644 --- a/backend/_example/memory_store/go.mod +++ b/backend/_example/memory_store/go.mod @@ -1,14 +1,15 @@ module github.com/umputun/remark/memory_store -go 1.12 +go 1.14 require ( github.com/go-pkgz/jrpc v0.1.0 - github.com/go-pkgz/lgr v0.6.3 + github.com/go-pkgz/lgr v0.7.0 github.com/jessevdk/go-flags v1.4.0 - github.com/pkg/errors v0.8.1 - github.com/stretchr/testify v1.4.0 - github.com/umputun/remark/backend v1.4.0 + github.com/pkg/errors v0.9.1 + github.com/rs/xid v1.2.1 + github.com/stretchr/testify v1.5.1 + github.com/umputun/remark/backend v1.5.0 ) replace github.com/umputun/remark/backend => ../../ diff --git a/backend/_example/memory_store/go.sum b/backend/_example/memory_store/go.sum index 1c2a3f90..ff3a466d 100644 --- a/backend/_example/memory_store/go.sum +++ b/backend/_example/memory_store/go.sum @@ -71,6 +71,8 @@ github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515 github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10= github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM= github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8= +github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8= +github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q= github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw= github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw= github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk= @@ -153,6 +155,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= @@ -177,6 +181,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE= github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE= @@ -280,6 +286,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -329,6 +336,7 @@ google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvx google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/backend/_example/memory_store/main.go b/backend/_example/memory_store/main.go index 07f2e85f..b7751c91 100644 --- a/backend/_example/memory_store/main.go +++ b/backend/_example/memory_store/main.go @@ -1,5 +1,5 @@ /* - * Copyright 2019 Umputun. All rights reserved. + * Copyright 2020 Umputun. All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ @@ -41,6 +41,7 @@ func main() { dataStore := accessor.NewMemData() adminStore := accessor.NewMemAdminStore(opts.Secret) + imgStore := accessor.NewMemImageStore() rpcServer := jrpc.Server{ API: opts.API, @@ -51,12 +52,13 @@ func main() { Logger: log.Default(), } - srv := server.NewRPC(dataStore, adminStore, &rpcServer) + srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer) admRec := accessor.AdminRec{ - SiteID: "remark", - IDs: []string{"dev_user"}, - Email: "admin@example.com", + SiteID: "remark", + Enabled: true, + IDs: []string{"dev_user"}, + Email: "admin@example.com", } adminStore.Set("remark", admRec) diff --git a/backend/_example/memory_store/server/image.go b/backend/_example/memory_store/server/image.go new file mode 100644 index 00000000..7bee7475 --- /dev/null +++ b/backend/_example/memory_store/server/image.go @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package server + +import ( + "context" + "encoding/base64" + "encoding/json" + "time" + + "github.com/go-pkgz/jrpc" +) + +func (s *RPC) imgSaveHndl(id uint64, params json.RawMessage) (rr jrpc.Response) { + var req [2]string + if err := json.Unmarshal(params, &req); err != nil { + return jrpc.Response{Error: err.Error()} + } + img, err := base64.StdEncoding.DecodeString(req[1]) + if err != nil { + return jrpc.Response{Error: err.Error()} + } + value, err := s.img.Save(req[0], img) + return jrpc.EncodeResponse(id, value, err) +} + +func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) { + var req [2]string + if err := json.Unmarshal(params, &req); err != nil { + return jrpc.Response{Error: err.Error()} + } + img, err := base64.StdEncoding.DecodeString(req[1]) + if err != nil { + return jrpc.Response{Error: err.Error()} + } + value, err := s.img.SaveWithID(req[0], img) + return jrpc.EncodeResponse(id, value, err) +} + +func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) { + var fileID string + if err := json.Unmarshal(params, &fileID); err != nil { + return jrpc.Response{Error: err.Error()} + } + value, err := s.img.Load(fileID) + return jrpc.EncodeResponse(id, value, err) +} + +func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) { + var fileID string + if err := json.Unmarshal(params, &fileID); err != nil { + return jrpc.Response{Error: err.Error()} + } + err := s.img.Commit(fileID) + return jrpc.EncodeResponse(id, nil, err) +} + +func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) { + var ttl time.Duration + if err := json.Unmarshal(params, &ttl); err != nil { + return jrpc.Response{Error: err.Error()} + } + err := s.img.Cleanup(context.TODO(), ttl) + return jrpc.EncodeResponse(id, nil, err) +} diff --git a/backend/_example/memory_store/server/image_test.go b/backend/_example/memory_store/server/image_test.go new file mode 100644 index 00000000..9c8adc38 --- /dev/null +++ b/backend/_example/memory_store/server/image_test.go @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package server + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/go-pkgz/jrpc" + "github.com/stretchr/testify/assert" + "github.com/umputun/remark/backend/app/store/image" +) + +// gopher png for test, from https://golang.org/src/image/png/example_test.go +const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" + + "+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" + + "73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" + + "/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" + + "JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" + + "8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" + + "ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" + + "MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" + + "MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" + + "rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" + + "rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" + + "dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" + + "yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" + + "LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" + + "11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" + + "A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" + + "yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" + + "1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" + + "nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==" + +func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } +func gopherPNGBytes() []byte { + img, _ := ioutil.ReadAll(gopherPNG()) + return img +} + +func TestRPC_imgSaveHndl(t *testing.T) { + _, port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + id, err := ri.Save("admin", gopherPNGBytes()) + assert.NoError(t, err) + assert.Contains(t, id, "admin/", "id contains username") + + err = ri.Commit(id) + assert.NoError(t, err) +} + +func TestRPC_imgSaveWithIDHndl(t *testing.T) { + _, port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + id, err := ri.SaveWithID("test_id", gopherPNGBytes()) + assert.NoError(t, err) + assert.Equal(t, id, "test_id") +} + +func TestRPC_imgLoadHndl(t *testing.T) { + _, port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + // save + id, err := ri.Save("admin", gopherPNGBytes()) + assert.NoError(t, err) + + // load + img, err := ri.Load(id) + assert.NoError(t, err) + assert.Equal(t, 1462, len(img)) + assert.Equal(t, gopherPNGBytes(), img) + + // commit + err = ri.Commit(id) + assert.NoError(t, err) + + // load after commit + img, err = ri.Load(id) + assert.NoError(t, err) + assert.Equal(t, 1462, len(img)) + assert.Equal(t, gopherPNGBytes(), img) +} + +func TestRPC_imgCommitHndlFail(t *testing.T) { + _, port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + err := ri.Commit("test_id") + assert.EqualError(t, err, "failed to commit test_id, not found in staging") +} + +func TestRPC_imgCleanupHndl(t *testing.T) { + _, port, teardown := prepTestStore(t) + defer teardown() + api := fmt.Sprintf("http://localhost:%d/test", port) + + ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}} + + // save + id, err := ri.Save("admin", gopherPNGBytes()) + assert.NoError(t, err) + + // load + _, err = ri.Load(id) + assert.NoError(t, err) + + // cleanup + err = ri.Cleanup(context.TODO(), time.Nanosecond) + assert.NoError(t, err) + + // load after cleanup should fail + _, err = ri.Load(id) + assert.Error(t, err) + assert.Contains(t, err.Error(), "image admin/") + assert.Contains(t, err.Error(), "not found") +} diff --git a/backend/_example/memory_store/server/rpc.go b/backend/_example/memory_store/server/rpc.go index b5ce3290..56ee9d04 100644 --- a/backend/_example/memory_store/server/rpc.go +++ b/backend/_example/memory_store/server/rpc.go @@ -8,6 +8,7 @@ package server import ( "github.com/go-pkgz/jrpc" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" @@ -19,11 +20,12 @@ type RPC struct { *jrpc.Server eng engine.Interface adm admin.Store + img image.Store } // NewRPC makes RPC instance and register handlers -func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC { - res := &RPC{eng: e, adm: a, Server: r} +func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC { + res := &RPC{eng: e, adm: a, img: i, Server: r} res.addHandlers() return res } @@ -52,4 +54,13 @@ func (s *RPC) addHandlers() { "enabled": s.admEnabledHndl, "event": s.admEventHndl, }) + + // image store handlers + s.Group("image", jrpc.HandlersGroup{ + "save": s.imgSaveHndl, + "save_with_id": s.imgSaveWithIDHndl, + "load": s.imgLoadHndl, + "commit": s.imgCommitHndl, + "cleanup": s.imgCleanupHndl, + }) } diff --git a/backend/_example/memory_store/server/rpc_test.go b/backend/_example/memory_store/server/rpc_test.go index d9d395ef..ca0af979 100644 --- a/backend/_example/memory_store/server/rpc_test.go +++ b/backend/_example/memory_store/server/rpc_test.go @@ -47,7 +47,8 @@ func waitForHTTPServerStart(port int) { func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) { mg := accessor.NewMemData() adm := accessor.NewMemAdminStore("secret") - s = NewRPC(mg, adm, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger}) + img := accessor.NewMemImageStore() + s = NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger}) admRec := accessor.AdminRec{ SiteID: "test-site", diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 034271c3..de5599e9 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -129,7 +129,7 @@ type StoreGroup struct { // ImageGroup defines options group for store pictures type ImageGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"rpc" default:"fs"` // nolint FS struct { Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"` Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"` @@ -138,9 +138,10 @@ type ImageGroup struct { Bolt struct { File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"` } `group:"bolt" namespace:"bolt" env-namespace:"bolt"` - MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"` - ResizeWidth int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"` - ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"` + MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"` + ResizeWidth int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"` + ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"` + RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"` } // AvatarGroup defines options group for avatar params @@ -567,39 +568,37 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { } func (s *ServerCommand) makePicturesStore() (*image.Service, error) { + imageServiceParams := image.ServiceParams{ + ImageAPI: s.RemarkURL + "/api/v1/picture/", + TTL: 5 * s.EditDuration, // add extra time to image TTL for staging + MaxSize: s.Image.MaxSize, + MaxHeight: s.Image.ResizeHeight, + MaxWidth: s.Image.ResizeWidth, + } switch s.Image.Type { case "bolt": - boltImageStore, err := image.NewBoltStorage( - s.Image.Bolt.File, - s.Image.MaxSize, - s.Image.ResizeHeight, - s.Image.ResizeWidth, - bolt.Options{}, - ) + boltImageStore, err := image.NewBoltStorage(s.Image.Bolt.File, bolt.Options{}) if err != nil { return nil, err } - return &image.Service{ - Store: boltImageStore, - ImageAPI: s.RemarkURL + "/api/v1/picture/", - TTL: 5 * s.EditDuration, // add extra time to image TTL for staging - }, nil + return image.NewService(boltImageStore, imageServiceParams), nil case "fs": if err := makeDirs(s.Image.FS.Path); err != nil { return nil, err } - return &image.Service{ - Store: &image.FileSystem{ - Location: s.Image.FS.Path, - Staging: s.Image.FS.Staging, - Partitions: s.Image.FS.Partitions, - MaxSize: s.Image.MaxSize, - MaxHeight: s.Image.ResizeHeight, - MaxWidth: s.Image.ResizeWidth, - }, - ImageAPI: s.RemarkURL + "/api/v1/picture/", - TTL: 5 * s.EditDuration, // add extra time to image TTL for staging - }, nil + return image.NewService(&image.FileSystem{ + Location: s.Image.FS.Path, + Staging: s.Image.FS.Staging, + Partitions: s.Image.FS.Partitions, + }, imageServiceParams), nil + case "rpc": + return image.NewService(&image.RPC{ + Client: jrpc.Client{ + API: s.Image.RPC.API, + Client: http.Client{Timeout: s.Image.RPC.TimeOut}, + AuthUser: s.Image.RPC.AuthUser, + AuthPasswd: s.Image.RPC.AuthPassword, + }}, imageServiceParams), nil } return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type) } @@ -769,7 +768,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator * VerificationSubject: s.Notify.Email.VerificationSubject, UnsubscribeURL: s.RemarkURL + "/email/unsubscribe.html", // TODO: uncomment after #560 frontend part is ready and URL is known - //SubscribeURL: s.RemarkURL + "/subscribe.html?token=", + // SubscribeURL: s.RemarkURL + "/subscribe.html?token=", TokenGenFn: func(userID, email, site string) (string, error) { claims := token.Claims{ Handshake: &token.Handshake{ID: userID + "::" + email}, diff --git a/backend/app/notify/telegram_test.go b/backend/app/notify/telegram_test.go index f29a22a4..7532b5ed 100644 --- a/backend/app/notify/telegram_test.go +++ b/backend/app/notify/telegram_test.go @@ -37,7 +37,9 @@ func TestTelegram_New(t *testing.T) { assert.EqualError(t, err, "unexpected telegram status code 404") _, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/") - assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused") + require.Error(t, err) + assert.Contains(t, err.Error(), "can't initialize telegram notifications") + assert.Contains(t, err.Error(), "dial tcp 127.0.0.1:4321: connect: connection refused") _, err = NewTelegram("good-token", "remark_test", 2*time.Second, "") assert.Error(t, err, "empty api url not allowed") diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index c5939729..580d9acd 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -425,7 +425,7 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) { CriticalScore: s.ScoreThresholds.Critical, PositiveScore: s.DataService.PositiveScore, ReadOnlyAge: s.ReadOnlyAge, - MaxImageSize: s.ImageService.Store.SizeLimit(), + MaxImageSize: s.ImageService.MaxSize, EmailNotifications: s.EmailNotifications, EmojiEnabled: s.EmojiEnabled, AnonVote: s.AnonVote, diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 83036242..05e22d8e 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -568,14 +568,14 @@ func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) { return } - file, header, err := r.FormFile("file") + file, _, err := r.FormFile("file") if err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal) return } defer func() { _ = file.Close() }() - id, err := s.imageService.Save(header.Filename, user.ID, file) + id, err := s.imageService.Save(user.ID, file) if err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal) return diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 9ef42a93..fc19eb81 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -901,13 +901,13 @@ func TestRest_CreateWithPictures(t *testing.T) { }() lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc) - imageService := svc.ImageService - imageService.Store = &image.FileSystem{ + imageService := image.NewService(&image.FileSystem{ Staging: "/tmp/remark42/images.staging", Location: "/tmp/remark42/images", - MaxSize: 2000, - } - imageService.TTL = 100 * time.Millisecond + }, image.ServiceParams{ + TTL: 100 * time.Millisecond, + MaxSize: 2000, + }) svc.privRest.imageService = imageService svc.ImageService = imageService diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index c7092800..b55a74e9 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -371,15 +371,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { Cache: memCache, WebRoot: tmp, RemarkURL: "https://demo.remark42.com", - ImageService: &image.Service{ - Store: &image.FileSystem{ - Location: tmp + "/pics-remark42", - Partitions: 100, - MaxSize: 10000, - Staging: tmp + "/pics-remark42/staging", - }, - TTL: time.Millisecond * 100, - }, + ImageService: image.NewService(&image.FileSystem{ + Location: tmp + "/pics-remark42", + Partitions: 100, + Staging: tmp + "/pics-remark42/staging", + }, image.ServiceParams{ + TTL: 100 * time.Millisecond, + MaxSize: 10000, + }), ImageProxy: &proxy.Image{}, ReadOnlyAge: 10, CommentFormatter: store.NewCommentFormatter(&proxy.Image{}), diff --git a/backend/app/rest/proxy/image_test.go b/backend/app/rest/proxy/image_test.go index 8b1ce808..9702c8e5 100644 --- a/backend/app/rest/proxy/image_test.go +++ b/backend/app/rest/proxy/image_test.go @@ -19,6 +19,33 @@ import ( "github.com/umputun/remark/backend/app/store/image" ) +// gopher png for test, from https://golang.org/src/image/png/example_test.go +const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" + + "+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" + + "73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" + + "/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" + + "JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" + + "8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" + + "ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" + + "MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" + + "MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" + + "rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" + + "rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" + + "dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" + + "yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" + + "LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" + + "11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" + + "A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" + + "yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" + + "1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" + + "nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==" + +func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } +func gopherPNGBytes() []byte { + img, _ := ioutil.ReadAll(gopherPNG()) + return img +} + func TestImage_Extract(t *testing.T) { tbl := []struct { @@ -81,7 +108,7 @@ func TestImage_Routes(t *testing.T) { resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL) require.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) - assert.Equal(t, "123", resp.Header["Content-Length"][0]) + assert.Equal(t, "1462", resp.Header["Content-Length"][0]) assert.Equal(t, "image/*", resp.Header["Content-Type"][0]) encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png")) @@ -101,7 +128,7 @@ func TestImage_RoutesCachingImage(t *testing.T) { CacheExternal: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", - ImageService: &image.Service{Store: &imageStore}, + ImageService: image.NewService(&imageStore, image.ServiceParams{MaxSize: 1500}), } ts := httptest.NewServer(http.HandlerFunc(img.Handler)) @@ -113,13 +140,13 @@ func TestImage_RoutesCachingImage(t *testing.T) { encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL)) imageStore.On("Load", mock.Anything).Once().Return(nil, nil) - imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Run(func(args mock.Arguments) { _, _ = ioutil.ReadAll(args.Get(1).(io.Reader)) }).Return("", nil) + imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Return("", nil) imageStore.On("Commit", mock.Anything).Once().Return(nil) resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL) require.Nil(t, err) assert.Equal(t, 200, resp.StatusCode) - assert.Equal(t, "123", resp.Header["Content-Length"][0]) + assert.Equal(t, "1462", resp.Header["Content-Length"][0]) assert.Equal(t, "image/*", resp.Header["Content-Type"][0]) imageStore.AssertCalled(t, "Load", mock.Anything) @@ -133,7 +160,7 @@ func TestImage_RoutesUsingCachedImage(t *testing.T) { CacheExternal: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", - ImageService: &image.Service{Store: &imageStore}, + ImageService: image.NewService(&imageStore, image.ServiceParams{}), } ts := httptest.NewServer(http.HandlerFunc(img.Handler)) @@ -216,9 +243,9 @@ func imgHTTPTestsServer(t *testing.T) *httptest.Server { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/image/img1.png" { t.Log("http img request", r.URL) - w.Header().Add("Content-Length", "123") + w.Header().Add("Content-Length", "1462") w.Header().Add("Content-Type", "image/png") - _, err := w.Write([]byte(fmt.Sprintf("%123s", "X"))) + _, err := w.Write(gopherPNGBytes()) assert.NoError(t, err) return } diff --git a/backend/app/store/admin/remote_test.go b/backend/app/store/admin/remote_test.go index f9d2722f..ae431ebe 100644 --- a/backend/app/store/admin/remote_test.go +++ b/backend/app/store/admin/remote_test.go @@ -96,6 +96,6 @@ func testServer(t *testing.T, req, resp string) *httptest.Server { require.NoError(t, err) assert.Equal(t, req, string(body)) t.Logf("req: %s", string(body)) - fmt.Fprintf(w, resp) + _, _ = fmt.Fprint(w, resp) })) } diff --git a/backend/app/store/image/bolt_store.go b/backend/app/store/image/bolt_store.go index f1d13a1c..5767f724 100644 --- a/backend/app/store/image/bolt_store.go +++ b/backend/app/store/image/bolt_store.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/binary" - "io" "path" "time" @@ -21,15 +20,12 @@ const insertTimeBktName = "insertTimestamps" // It uses 3 buckets to manage images data. // Two buckets contains image data (staged and committed images). Third bucket holds insertion timestamps. type Bolt struct { - fileName string - db *bolt.DB - MaxSize int - MaxHeight int - MaxWidth int + fileName string + db *bolt.DB } // NewBoltStorage create bolt image store -func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, options bolt.Options) (*Bolt, error) { +func NewBoltStorage(fileName string, options bolt.Options) (*Bolt, error) { db, err := bolt.Open(fileName, 0600, &options) if err != nil { return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName) @@ -51,44 +47,34 @@ func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, o return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName) } return &Bolt{ - db: db, - fileName: fileName, - MaxSize: maxSize, - MaxHeight: maxHeight, - MaxWidth: maxWidth, + db: db, + fileName: fileName, }, nil } // SaveWithID saves data from a reader, for given id -func (b *Bolt) SaveWithID(id string, r io.Reader) (string, error) { - data, err := readAndValidateImage(r, b.MaxSize) - if err != nil { - return "", errors.Wrapf(err, "can't load image with ID %s", id) - } - - data = resize(data, b.MaxWidth, b.MaxHeight) - - err = b.db.Update(func(tx *bolt.Tx) error { - if err = tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), data); err != nil { +func (b *Bolt) SaveWithID(id string, img []byte) (string, error) { + err := b.db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), img); err != nil { return errors.Wrapf(err, "can't put to bucket with %s", id) } tsBuf := &bytes.Buffer{} - if err = binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil { + if err := binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil { return errors.Wrapf(err, "can't serialize timestamp for %s", id) } - if err = tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil { + if err := tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil { return errors.Wrapf(err, "can't put to bucket with %s", id) } - return err + return nil }) return id, err } // Save data from reader to staging bucket in DB -func (b *Bolt) Save(_ string, userID string, r io.Reader) (id string, err error) { +func (b *Bolt) Save(userID string, img []byte) (id string, err error) { id = path.Join(userID, guid()) - return b.SaveWithID(id, r) + return b.SaveWithID(id, img) } // Commit file stored in staging bucket by copying it to permanent bucket @@ -130,7 +116,7 @@ func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error { err := b.db.Update(func(tx *bolt.Tx) error { c := tx.Bucket([]byte(insertTimeBktName)).Cursor() - idsToRemove := [][]byte{} + var idsToRemove [][]byte for id, tsData := c.First(); id != nil; id, tsData = c.Next() { var ts int64 @@ -161,8 +147,3 @@ func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error { }) return err } - -// SizeLimit returns max size of allowed image -func (b *Bolt) SizeLimit() int { - return b.MaxSize -} diff --git a/backend/app/store/image/bolt_store_test.go b/backend/app/store/image/bolt_store_test.go index 27e01fed..89748f6f 100644 --- a/backend/app/store/image/bolt_store_test.go +++ b/backend/app/store/image/bolt_store_test.go @@ -18,7 +18,7 @@ func TestBoltStore_SaveCommit(t *testing.T) { svc, teardown := prepareBoltImageStorageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) assert.NoError(t, err) assert.Contains(t, id, "user1") t.Log(id) @@ -48,7 +48,7 @@ func TestBoltStore_LoadAfterSave(t *testing.T) { svc, teardown := prepareBoltImageStorageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) assert.NoError(t, err) assert.Contains(t, id, "user1") t.Log(id) @@ -56,6 +56,7 @@ func TestBoltStore_LoadAfterSave(t *testing.T) { data, err := svc.Load(id) assert.NoError(t, err) assert.Equal(t, 1462, len(data)) + assert.Equal(t, gopherPNGBytes(), data) _, err = svc.Load("abcd") assert.Error(t, err) @@ -66,7 +67,7 @@ func TestBoltStore_Cleanup(t *testing.T) { defer teardown() save := func(file string, user string) (id string) { - id, err := svc.Save(file, user, gopherPNG()) + id, err := svc.Save(user, gopherPNGBytes()) require.NoError(t, err) checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error { @@ -133,7 +134,7 @@ func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) { loc, err := ioutil.TempDir("", "test_image_r42") require.NoError(t, err, "failed to make temp dir") - svc, err = NewBoltStorage(path.Join(loc, "picture.db"), 1500, 0, 0, bolt.Options{}) + svc, err = NewBoltStorage(path.Join(loc, "picture.db"), bolt.Options{}) assert.NoError(t, err, "new bolt storage") teardown = func() { diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go index 80377d6b..a7177f00 100644 --- a/backend/app/store/image/fs_store.go +++ b/backend/app/store/image/fs_store.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "hash/crc64" - "io" "io/ioutil" "math" "os" @@ -23,10 +22,7 @@ import ( type FileSystem struct { Location string Staging string - MaxSize int Partitions int - MaxHeight int - MaxWidth int crc struct { *crc64.Table @@ -37,36 +33,26 @@ type FileSystem struct { } // SaveWithID saves data from a reader, with given id -func (f *FileSystem) SaveWithID(id string, r io.Reader) (string, error) { - data, err := readAndValidateImage(r, f.MaxSize) - if err != nil { - return "", errors.Wrapf(err, "can't load image with ID %s", id) - } - - data = resize(data, f.MaxWidth, f.MaxHeight) +func (f *FileSystem) SaveWithID(id string, img []byte) (string, error) { dst := f.location(f.Staging, id) - if err = os.MkdirAll(path.Dir(dst), 0700); err != nil { + if err := os.MkdirAll(path.Dir(dst), 0700); err != nil { return "", errors.Wrap(err, "can't make image directory") } - if err = ioutil.WriteFile(dst, data, 0600); err != nil { + if err := ioutil.WriteFile(dst, img, 0600); err != nil { return "", errors.Wrapf(err, "can't write image file with id %s", id) } - log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(data)) + log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(img)) return id, nil } -// Save data from a reader for given file name to local FS, staging directory. Returns id as user/uuid +// Save data from a reader to local FS, staging directory. Returns id as user/uuid // Files partitioned across multiple subdirectories, and the final path includes part, i.e. /location/user1/03/123-4567 -func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) { +func (f *FileSystem) Save(userID string, img []byte) (id string, err error) { tempId := path.Join(userID, guid()) // make id as user/uuid - id, err = f.SaveWithID(tempId, r) - if err != nil { - err = errors.Wrapf(err, "can't save image file %s", fileName) - } - return id, err + return f.SaveWithID(tempId, img) } // Commit file stored in staging location by moving it to permanent location @@ -135,11 +121,6 @@ func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error { return errors.Wrap(err, "failed to cleanup images") } -// SizeLimit returns max size of allowed image -func (f *FileSystem) SizeLimit() int { - return f.MaxSize -} - // location gets full path for id by adding partition to the final path in order to keep files in different subdirectories // and avoid too many files in a single place. // the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png. diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go index d2077d81..134fc98a 100644 --- a/backend/app/store/image/fs_store_test.go +++ b/backend/app/store/image/fs_store_test.go @@ -39,12 +39,16 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU "nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==" func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } +func gopherPNGBytes() []byte { + img, _ := ioutil.ReadAll(gopherPNG()) + return img +} func TestFsStore_Save(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) assert.NoError(t, err) assert.Contains(t, id, "user1/") t.Log(id) @@ -56,78 +60,32 @@ func TestFsStore_Save(t *testing.T) { assert.Equal(t, 1462, len(data)) } -func TestFsStore_SaveWithResize(t *testing.T) { - svc, teardown := prepareImageTest(t) - defer teardown() - svc.MaxWidth, svc.MaxHeight = 32, 32 - - id, err := svc.Save("file1.png", "user1", gopherPNG()) - assert.NoError(t, err) - assert.Contains(t, id, "user1/") - t.Log(id) - - img := svc.location(svc.Staging, id) - t.Log(img) - data, err := ioutil.ReadFile(img) - assert.NoError(t, err) - assert.Equal(t, 1135, len(data)) -} - -func TestFsStore_SaveWithResizeJpeg(t *testing.T) { - svc, teardown := prepareImageTest(t) - defer teardown() - svc.MaxWidth, svc.MaxHeight = 400, 300 - svc.MaxSize = 32000 - - fh, err := os.Open("testdata/circles.jpg") - defer func() { assert.NoError(t, fh.Close()) }() - assert.NoError(t, err) - id, err := svc.Save("circles.jpg", "user1", fh) - assert.NoError(t, err) - assert.Contains(t, id, "user1/") - t.Log(id) - - img := svc.location(svc.Staging, id) - t.Log(img) - data, err := ioutil.ReadFile(img) - assert.NoError(t, err) - assert.Equal(t, 10918, len(data)) -} - func TestFsStore_SaveNoResizeJpeg(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - svc.MaxWidth, svc.MaxHeight = 1400, 1300 - svc.MaxSize = 32000 fh, err := os.Open("testdata/circles.jpg") defer func() { assert.NoError(t, fh.Close()) }() assert.NoError(t, err) - id, err := svc.Save("circles.jpg", "user1", fh) + img, err := ioutil.ReadAll(fh) + assert.NoError(t, err) + id, err := svc.Save("user1", img) assert.NoError(t, err) assert.Contains(t, id, "user1/") t.Log(id) - img := svc.location(svc.Staging, id) - t.Log(img) - data, err := ioutil.ReadFile(img) + imgPath := svc.location(svc.Staging, id) + t.Log(imgPath) + data, err := ioutil.ReadFile(imgPath) assert.NoError(t, err) assert.Equal(t, 23983, len(data)) } -func TestFsStore_WrongFormat(t *testing.T) { - svc, teardown := prepareImageTest(t) - defer teardown() - - _, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image")) - assert.Error(t, err) -} - func TestFsStore_SaveAndCommit(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) require.NoError(t, err) err = svc.Commit(id) require.NoError(t, err) @@ -143,27 +101,19 @@ func TestFsStore_SaveAndCommit(t *testing.T) { assert.Equal(t, 1462, len(data)) } -func TestFsStore_SaveTooLarge(t *testing.T) { - svc, teardown := prepareImageTest(t) - defer teardown() - svc.MaxSize = 2000 - _, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG())) - assert.Error(t, err) - assert.Contains(t, err.Error(), "is too large") -} - func TestFsStore_LoadAfterSave(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("blah_ff1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) assert.NoError(t, err) t.Log(id) data, err := svc.Load(id) assert.NoError(t, err) assert.Equal(t, 1462, len(data)) + assert.Equal(t, gopherPNGBytes(), data) _, err = svc.Load("abcd") assert.Error(t, err) } @@ -173,7 +123,7 @@ func TestFsStore_LoadAfterCommit(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("blah_ff1.png", "user1", gopherPNG()) + id, err := svc.Save("user1", gopherPNGBytes()) assert.NoError(t, err) t.Log(id) err = svc.Commit(id) @@ -235,7 +185,7 @@ func TestFsStore_Cleanup(t *testing.T) { defer teardown() save := func(file string, user string) (path string) { - id, err := svc.Save(file, user, gopherPNG()) + id, err := svc.Save(user, gopherPNGBytes()) require.NoError(t, err) img := svc.location(svc.Staging, id) data, err := ioutil.ReadFile(img) @@ -292,7 +242,6 @@ func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) { Location: loc, Staging: staging, Partitions: 100, - MaxSize: 1500, } teardown = func() { diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index 7a55e5f6..5767c4fa 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -31,26 +31,35 @@ import ( // It also provides async Submit with func param retrieving all submitting ids. // Submitted ids committed (i.e. moved from staging to final) on TTL expiration. type Service struct { - Store - TTL time.Duration // for how long file allowed on staging - ImageAPI string // image api matching path + ServiceParams + store Store wg sync.WaitGroup submitCh chan submitReq once sync.Once term int32 // term value used atomically to detect emergency termination } +// ServiceParams contains externally adjustable parameters of Service +type ServiceParams struct { + TTL time.Duration // for how long file allowed on staging + ImageAPI string // image api matching path + MaxSize int + MaxHeight int + MaxWidth int +} + // To regenerate mock run from this directory: // sh -c "mockery -inpkg -name Store -print > /tmp/image-mock.tmp && mv /tmp/image-mock.tmp image_mock.go" // Store defines interface for saving and loading pictures. -// Declares two-stage save with Commit. Save stores to staging area and Commit moves to the final location +// Declares two-stage save with Commit. Save stores to staging area and Commit moves to the final location. +// Two-stage commit scheme is used for not storing images which are uploaded but later never used in the comments, +// e.g. when somebody uploaded a picture but did not sent the comment. type Store interface { - Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored (staging) image - SaveWithID(id string, r io.Reader) (string, error) // store image for passed id to staging - Load(id string) ([]byte, error) // load image by ID. Caller has to close the reader. - SizeLimit() int // max image size + Save(userID string, img []byte) (id string, err error) // get name and reader and returns ID of stored (staging) image + SaveWithID(id string, img []byte) (string, error) // store image for passed id to staging + Load(id string) ([]byte, error) // load image by ID. Caller has to close the reader. Commit(id string) error // move image from staging to permanent Cleanup(ctx context.Context, ttl time.Duration) error // run removal loop for old images on staging @@ -63,6 +72,10 @@ type submitReq struct { TS time.Time } +func NewService(s Store, p ServiceParams) *Service { + return &Service{ServiceParams: p, store: s} +} + // Submit multiple ids via function for delayed commit func (s *Service) Submit(idsFn func() []string) { if idsFn == nil || s == nil { @@ -81,7 +94,7 @@ func (s *Service) Submit(idsFn func() []string) { time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close) } for _, id := range req.idsFn() { - if err := s.Commit(id); err != nil { + if err := s.store.Commit(id); err != nil { log.Printf("[WARN] failed to commit image %s", id) } } @@ -127,7 +140,7 @@ func (s *Service) Cleanup(ctx context.Context) { log.Printf("[INFO] cleanup terminated, %v", ctx.Err()) return case <-time.After(s.TTL / 2): // cleanup call on every 1/2 TTL - if err := s.Store.Cleanup(ctx, s.TTL); err != nil { + if err := s.store.Cleanup(ctx, s.TTL); err != nil { log.Printf("[WARN] failed to cleanup, %v", err) } } @@ -151,6 +164,40 @@ func (s *Service) Close() { s.wg.Wait() } +// Load wraps storage Load function. +func (s *Service) Load(id string) ([]byte, error) { + return s.store.Load(id) +} + +// Save wraps storage Save function, validating and resizing the image before calling it. +func (s *Service) Save(userID string, r io.Reader) (id string, err error) { + img, err := s.prepareImage(r) + if err != nil { + return "", err + } + return s.store.Save(userID, img) +} + +// SaveWithID wraps storage SaveWithID function, validating and resizing the image before calling it. +func (s *Service) SaveWithID(id string, r io.Reader) (string, error) { + img, err := s.prepareImage(r) + if err != nil { + return "", err + } + return s.store.SaveWithID(id, img) +} + +// prepareImage calls readAndValidateImage and resize on provided image. +func (s *Service) prepareImage(r io.Reader) ([]byte, error) { + data, err := readAndValidateImage(r, s.MaxSize) + if err != nil { + return nil, errors.Wrapf(err, "can't load image") + } + + data = resize(data, s.MaxWidth, s.MaxHeight) + return data, nil +} + // resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of // the biggest side (width or height) preserving aspect ratio. // Returns original data if resizing is not needed or failed. diff --git a/backend/app/store/image/image_mock.go b/backend/app/store/image/image_mock.go index 8c23293c..e32fc262 100644 --- a/backend/app/store/image/image_mock.go +++ b/backend/app/store/image/image_mock.go @@ -3,7 +3,6 @@ package image import context "context" -import io "io" import mock "github.com/stretchr/testify/mock" import time "time" @@ -63,20 +62,20 @@ func (_m *MockStore) Load(id string) ([]byte, error) { return r0, r1 } -// Save provides a mock function with given fields: fileName, userID, r -func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) { - ret := _m.Called(fileName, userID, r) +// Save provides a mock function with given fields: userID, img +func (_m *MockStore) Save(userID string, img []byte) (string, error) { + ret := _m.Called(userID, img) var r0 string - if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok { - r0 = rf(fileName, userID, r) + if rf, ok := ret.Get(0).(func(string, []byte) string); ok { + r0 = rf(userID, img) } else { r0 = ret.Get(0).(string) } var r1 error - if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok { - r1 = rf(fileName, userID, r) + if rf, ok := ret.Get(1).(func(string, []byte) error); ok { + r1 = rf(userID, img) } else { r1 = ret.Error(1) } @@ -84,37 +83,23 @@ func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, return r0, r1 } -// SaveWithID provides a mock function with given fields: id, r -func (_m *MockStore) SaveWithID(id string, r io.Reader) (string, error) { - ret := _m.Called(id, r) +// SaveWithID provides a mock function with given fields: id, img +func (_m *MockStore) SaveWithID(id string, img []byte) (string, error) { + ret := _m.Called(id, img) var r0 string - if rf, ok := ret.Get(0).(func(string, io.Reader) string); ok { - r0 = rf(id, r) + if rf, ok := ret.Get(0).(func(string, []byte) string); ok { + r0 = rf(id, img) } else { r0 = ret.Get(0).(string) } var r1 error - if rf, ok := ret.Get(1).(func(string, io.Reader) error); ok { - r1 = rf(id, r) + if rf, ok := ret.Get(1).(func(string, []byte) error); ok { + r1 = rf(id, img) } else { r1 = ret.Error(1) } return r0, r1 } - -// SizeLimit provides a mock function with given fields: -func (_m *MockStore) SizeLimit() int { - ret := _m.Called() - - var r0 int - if rf, ok := ret.Get(0).(func() int); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int) - } - - return r0 -} diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 5d55789c..27ed6451 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -4,8 +4,11 @@ import ( "bytes" "context" "image" + "io" "io/ioutil" + "os" "strconv" + "strings" "testing" "time" @@ -14,8 +17,68 @@ import ( "github.com/stretchr/testify/require" ) +func TestService_SaveAndLoad(t *testing.T) { + store := MockStore{} + svc := NewService(&store, ServiceParams{MaxSize: 1500, MaxWidth: 32, MaxHeight: 32}) + + store.On("Save", "user1", mock.Anything).Return("user1/test_id", nil) + id, err := svc.Save("user1", gopherPNG()) + assert.NoError(t, err) + assert.Equal(t, "user1/test_id", id) + + store.On("SaveWithID", "test_id", mock.Anything).Return("test_id", nil) + id, err = svc.SaveWithID("test_id", gopherPNG()) + assert.NoError(t, err) + assert.Equal(t, "test_id", id) + + store.On("Load", "test_id", mock.Anything).Return(nil, nil) + img, err := svc.Load("test_id") + assert.NoError(t, err) + assert.Nil(t, img) +} + +func TestService_Resize(t *testing.T) { + img, err := readAndValidateImage(gopherPNG(), 1500) + assert.NoError(t, err) + assert.Equal(t, 1462, len(img)) + + img = resize(img, 32, 32) + assert.Equal(t, 1135, len(img)) +} + +func TestService_ResizeJpeg(t *testing.T) { + fh, err := os.Open("testdata/circles.jpg") + defer func() { assert.NoError(t, fh.Close()) }() + assert.NoError(t, err) + + img, err := readAndValidateImage(fh, 32000) + assert.NoError(t, err) + assert.Equal(t, 23983, len(img)) + + img = resize(img, 400, 300) + assert.Equal(t, 10918, len(img)) +} + +func TestService_SaveTooLarge(t *testing.T) { + svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}} + svc.MaxSize = 2000 + _, err := svc.Save("user2", io.MultiReader(gopherPNG(), gopherPNG())) + assert.Error(t, err) + assert.Contains(t, err.Error(), "is too large") + _, err = svc.SaveWithID("test_id", io.MultiReader(gopherPNG(), gopherPNG())) + assert.Error(t, err) + assert.Contains(t, err.Error(), "is too large") +} + +func TestService_WrongFormat(t *testing.T) { + svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}} + + _, err := svc.Save("user1", strings.NewReader("blah blah bad image")) + assert.Error(t, err) +} + func TestService_ExtractPictures(t *testing.T) { - svc := Service{ImageAPI: "/blah/"} + svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}} html := `blah foo xyz

123

` ids, err := svc.ExtractPictures(html) @@ -26,7 +89,7 @@ func TestService_ExtractPictures(t *testing.T) { } func TestService_ExtractPictures2(t *testing.T) { - svc := Service{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"} + svc := Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}} html := "

TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." + "

\n\n

Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:

\n\n

\"bjtr0-201906-08110846-i324c.png\"/

\n\n

По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. github.com/go-pkgz/auth v0.5.2.

\n" ids, err := svc.ExtractPictures(html) @@ -39,7 +102,7 @@ func TestService_Cleanup(t *testing.T) { store := MockStore{} store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil) - svc := Service{Store: &store, TTL: 100 * time.Millisecond} + svc := Service{store: &store, ServiceParams: ServiceParams{TTL: 100 * time.Millisecond}} ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549) defer cancel() svc.Cleanup(ctx) @@ -49,7 +112,7 @@ func TestService_Cleanup(t *testing.T) { func TestService_Submit(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) - svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} + svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Millisecond * 100}} svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) svc.Submit(func() []string { return []string{"id4", "id5"} }) svc.Submit(nil) @@ -61,7 +124,7 @@ func TestService_Submit(t *testing.T) { func TestService_Close(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) - svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500} + svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Millisecond * 500}} svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) svc.Submit(func() []string { return []string{"id4", "id5"} }) svc.Submit(nil) @@ -72,7 +135,7 @@ func TestService_Close(t *testing.T) { func TestService_SubmitDelay(t *testing.T) { store := MockStore{} store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil) - svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} + svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", TTL: time.Millisecond * 100}} svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) time.Sleep(150 * time.Millisecond) // let first batch to pass TTL svc.Submit(func() []string { return []string{"id4", "id5"} }) diff --git a/backend/app/store/image/remote_store.go b/backend/app/store/image/remote_store.go new file mode 100644 index 00000000..b374fdf1 --- /dev/null +++ b/backend/app/store/image/remote_store.go @@ -0,0 +1,58 @@ +package image + +import ( + "context" + "encoding/base64" + "encoding/json" + "io/ioutil" + "strings" + "time" + + "github.com/go-pkgz/jrpc" +) + +// RPC implements remote engine and delegates all Calls to remote http server +type RPC struct { + jrpc.Client +} + +func (r *RPC) Save(userID string, img []byte) (id string, err error) { + resp, err := r.Call("image.save", userID, img) + if err != nil { + return "", err + } + err = json.Unmarshal(*resp.Result, &id) + return id, err +} + +func (r *RPC) SaveWithID(id string, img []byte) (string, error) { + resp, err := r.Call("image.save_with_id", id, img) + if err != nil { + return "", err + } + var newID string + err = json.Unmarshal(*resp.Result, &newID) + return newID, err +} + +func (r *RPC) Load(id string) ([]byte, error) { + resp, err := r.Call("image.load", id) + if err != nil { + return nil, err + } + var rawImg string + if err = json.Unmarshal(*resp.Result, &rawImg); err != nil { + return nil, err + } + return ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg))) +} + +func (r *RPC) Commit(id string) error { + _, err := r.Call("image.commit", id) + return err +} + +func (r *RPC) Cleanup(_ context.Context, ttl time.Duration) error { + _, err := r.Call("image.cleanup", ttl) + return err +} diff --git a/backend/app/store/image/remote_store_test.go b/backend/app/store/image/remote_store_test.go new file mode 100644 index 00000000..e22d8752 --- /dev/null +++ b/backend/app/store/image/remote_store_test.go @@ -0,0 +1,90 @@ +package image + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-pkgz/jrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemote_Save(t *testing.T) { + ts := testServer(t, fmt.Sprintf(`{"method":"image.save","params":["admin","%s"],"id":1}`, gopher), + `{"result":"12345","id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Save("admin", gopherPNGBytes()) + assert.NoError(t, err) + assert.Equal(t, "12345", res) +} + +func TestRemote_SaveWithID(t *testing.T) { + ts := testServer(t, fmt.Sprintf(`{"method":"image.save_with_id","params":["54321","%s"],"id":1}`, gopher), + `{"result":"12345","id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.SaveWithID("54321", gopherPNGBytes()) + assert.NoError(t, err) + assert.Equal(t, "12345", res) +} + +func TestRemote_Load(t *testing.T) { + ts := testServer(t, `{"method":"image.load","params":"54321","id":1}`, + fmt.Sprintf(`{"result":"%v","id":1}`, gopher)) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Load("54321") + assert.NoError(t, err) + assert.Equal(t, gopherPNGBytes(), res) +} + +func TestRemote_Commit(t *testing.T) { + ts := testServer(t, `{"method":"image.commit","params":"gopher_id","id":1}`, `{"id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + err := c.Commit("gopher_id") + assert.NoError(t, err) +} + +func TestRemote_Cleanup(t *testing.T) { + ts := testServer(t, `{"method":"image.cleanup","params":60000000000,"id":1}`, `{"id":1}`) + defer ts.Close() + c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + err := c.Cleanup(context.TODO(), time.Minute) + assert.NoError(t, err) +} + +func testServer(t *testing.T, req, resp string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, req, string(body)) + _, _ = fmt.Fprint(w, resp) + })) +} diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index bf15b6dc..b237f600 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -1278,7 +1278,7 @@ func TestService_submitImages(t *testing.T) { mockStore := image.MockStore{} mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil) - imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50} + imgSvc := image.NewService(&mockStore, image.ServiceParams{TTL: 50 * time.Millisecond * 50}) // two comments for https://radio-t.com eng, teardown := prepStoreEngine(t) diff --git a/backend/go.mod b/backend/go.mod index 85f047b1..dfe89f22 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module github.com/umputun/remark/backend -go 1.13 +go 1.14 require ( cloud.google.com/go v0.47.0 // indirect diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 3fd39085..1aa66b6b 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -1,10 +1,16 @@ # cloud.google.com/go v0.47.0 +## explicit cloud.google.com/go/compute/metadata # github.com/Depado/bfchroma v1.2.0 +## explicit github.com/Depado/bfchroma # github.com/PuerkitoBio/goquery v1.5.0 +## explicit github.com/PuerkitoBio/goquery +# github.com/ajg/form v1.5.1 +## explicit # github.com/alecthomas/chroma v0.6.0 +## explicit github.com/alecthomas/chroma github.com/alecthomas/chroma/formatters/html github.com/alecthomas/chroma/lexers @@ -36,8 +42,10 @@ github.com/alecthomas/chroma/lexers/x github.com/alecthomas/chroma/lexers/y github.com/alecthomas/chroma/styles # github.com/andybalholm/cascadia v1.1.0 +## explicit github.com/andybalholm/cascadia # github.com/coreos/bbolt v1.3.3 +## explicit github.com/coreos/bbolt # github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/danwakefield/fnmatch @@ -47,25 +55,36 @@ github.com/davecgh/go-spew/spew github.com/dghubble/oauth1 github.com/dghubble/oauth1/twitter # github.com/dgrijalva/jwt-go v3.2.0+incompatible +## explicit github.com/dgrijalva/jwt-go # github.com/didip/tollbooth v4.0.2+incompatible +## explicit github.com/didip/tollbooth github.com/didip/tollbooth/errors github.com/didip/tollbooth/libstring github.com/didip/tollbooth/limiter # github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d +## explicit github.com/didip/tollbooth_chi # github.com/dlclark/regexp2 v1.1.6 github.com/dlclark/regexp2 github.com/dlclark/regexp2/syntax +# github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 +## explicit +# github.com/gavv/httpexpect v2.0.0+incompatible +## explicit # github.com/go-chi/chi v4.0.2+incompatible +## explicit github.com/go-chi/chi github.com/go-chi/chi/middleware # github.com/go-chi/cors v1.0.0 +## explicit github.com/go-chi/cors # github.com/go-chi/render v1.0.1 +## explicit github.com/go-chi/render # github.com/go-pkgz/auth v0.9.0 +## explicit github.com/go-pkgz/auth github.com/go-pkgz/auth/avatar github.com/go-pkgz/auth/logger @@ -74,18 +93,24 @@ github.com/go-pkgz/auth/provider github.com/go-pkgz/auth/provider/sender github.com/go-pkgz/auth/token # github.com/go-pkgz/jrpc v0.1.0 +## explicit github.com/go-pkgz/jrpc # github.com/go-pkgz/lcw v0.5.0 +## explicit github.com/go-pkgz/lcw # github.com/go-pkgz/lgr v0.6.3 +## explicit github.com/go-pkgz/lgr # github.com/go-pkgz/repeater v1.1.3 +## explicit github.com/go-pkgz/repeater github.com/go-pkgz/repeater/strategy # github.com/go-pkgz/rest v1.4.1 +## explicit github.com/go-pkgz/rest github.com/go-pkgz/rest/logger # github.com/go-pkgz/syncs v1.1.1 +## explicit github.com/go-pkgz/syncs # github.com/go-redis/redis/v7 v7.0.0-beta.4 github.com/go-redis/redis/v7 @@ -102,49 +127,91 @@ github.com/golang/protobuf/proto # github.com/golang/snappy v0.0.1 github.com/golang/snappy # github.com/google/uuid v1.1.1 +## explicit github.com/google/uuid +# github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f +## explicit # github.com/gorilla/feeds v1.1.1 +## explicit github.com/gorilla/feeds +# github.com/gorilla/websocket v1.4.0 +## explicit # github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/errwrap # github.com/hashicorp/go-multierror v1.0.0 +## explicit github.com/hashicorp/go-multierror # github.com/hashicorp/golang-lru v0.5.3 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru # github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc +## explicit github.com/jessevdk/go-flags +# github.com/klauspost/compress v1.7.6 +## explicit +# github.com/klauspost/cpuid v1.2.1 +## explicit # github.com/kyokomi/emoji v2.1.0+incompatible +## explicit github.com/kyokomi/emoji +# github.com/mattn/go-colorable v0.1.2 +## explicit +# github.com/mattn/go-isatty v0.0.9 +## explicit # github.com/microcosm-cc/bluemonday v1.0.2 +## explicit github.com/microcosm-cc/bluemonday # github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 github.com/nullrocks/identicon +# github.com/onsi/ginkgo v1.9.0 +## explicit +# github.com/onsi/gomega v1.6.0 +## explicit # github.com/patrickmn/go-cache v2.1.0+incompatible +## explicit github.com/patrickmn/go-cache # github.com/pkg/errors v0.8.1 +## explicit github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib # github.com/rakyll/statik v0.1.6 +## explicit github.com/rakyll/statik/fs # github.com/rs/xid v1.2.1 +## explicit github.com/rs/xid # github.com/russross/blackfriday/v2 v2.0.1 +## explicit github.com/russross/blackfriday/v2 # github.com/shurcooL/sanitized_anchor_name v1.0.0 github.com/shurcooL/sanitized_anchor_name +# github.com/smartystreets/assertions v1.0.1 +## explicit # github.com/stretchr/objx v0.2.0 +## explicit github.com/stretchr/objx # github.com/stretchr/testify v1.4.0 +## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require +# github.com/tidwall/buntdb v1.1.0 +## explicit +# github.com/tidwall/gjson v1.3.2 +## explicit +# github.com/valyala/fasthttp v1.4.0 +## explicit # github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/xdg/scram # github.com/xdg/stringprep v1.0.0 github.com/xdg/stringprep +# github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b +## explicit +# github.com/xeipuuv/gojsonschema v1.1.0 +## explicit # go.mongodb.org/mongo-driver v1.1.2 +## explicit go.mongodb.org/mongo-driver/bson go.mongodb.org/mongo-driver/bson/bsoncodec go.mongodb.org/mongo-driver/bson/bsonrw @@ -175,13 +242,16 @@ go.mongodb.org/mongo-driver/x/mongo/driver/topology go.mongodb.org/mongo-driver/x/mongo/driver/uuid go.mongodb.org/mongo-driver/x/mongo/driver/wiremessage # golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 +## explicit golang.org/x/crypto/acme golang.org/x/crypto/acme/autocert golang.org/x/crypto/pbkdf2 # golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 +## explicit golang.org/x/image/draw golang.org/x/image/math/f64 # golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a +## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/html @@ -197,8 +267,10 @@ golang.org/x/oauth2/jws golang.org/x/oauth2/jwt golang.org/x/oauth2/yandex # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +## explicit golang.org/x/sync/semaphore # golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 +## explicit golang.org/x/sys/unix # golang.org/x/text v0.3.2 golang.org/x/text/secure/bidirule @@ -206,8 +278,10 @@ golang.org/x/text/transform golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm # golang.org/x/time v0.0.0-20191024005414-555d28b269f0 +## explicit golang.org/x/time/rate # google.golang.org/appengine v1.6.5 +## explicit google.golang.org/appengine google.golang.org/appengine/internal google.golang.org/appengine/internal/app_identity @@ -219,8 +293,10 @@ google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # gopkg.in/oauth2.v3 v3.11.0 +## explicit gopkg.in/oauth2.v3 gopkg.in/oauth2.v3/errors gopkg.in/oauth2.v3/server # gopkg.in/yaml.v2 v2.2.4 +## explicit gopkg.in/yaml.v2 diff --git a/frontend/app/common/local-storage.test.ts b/frontend/app/common/local-storage.test.ts new file mode 100644 index 00000000..b3ee40d9 --- /dev/null +++ b/frontend/app/common/local-storage.test.ts @@ -0,0 +1,95 @@ +import { setJsonItem, getJsonItem, updateJsonItem } from './local-storage'; + +const LS_KEY = 'test'; + +describe('getJsonItem', () => { + afterAll(() => { + localStorage.clear(); + }); + it('should set json to empty localStorage', () => { + setJsonItem>(LS_KEY, {}); + expect(localStorage.getItem(LS_KEY)).toBe('{}'); + }); + + it('should update json in localStoeage', () => { + setJsonItem(LS_KEY, []); + expect(localStorage.getItem(LS_KEY)).toBe('[]'); + }); +}); + +describe('setJsonItem', () => { + let consoleSpy: jest.SpyInstance; + + beforeEach(() => { + consoleSpy = jest.spyOn(console, 'error').mockImplementation(); + }); + afterEach(() => { + localStorage.clear(); + }); + + it('should return null when localStorage is empty', () => { + expect(getJsonItem(LS_KEY)).toBe(null); + }); + + it('should return value of key', () => { + localStorage.setItem(LS_KEY, JSON.stringify({})); + expect(getJsonItem(LS_KEY)).toEqual({}); + + localStorage.setItem(LS_KEY, JSON.stringify([])); + expect(getJsonItem(LS_KEY)).toEqual([]); + + localStorage.setItem(LS_KEY, JSON.stringify(null)); + expect(getJsonItem(LS_KEY)).toBe(null); + + localStorage.setItem(LS_KEY, JSON.stringify(1)); + expect(getJsonItem(LS_KEY)).toBe(1); + + localStorage.setItem(LS_KEY, JSON.stringify(1)); + expect(getJsonItem(LS_KEY)).toBe(1); + }); + + it('should return `null` if value in localStorage is not JSON', () => { + localStorage.setItem(LS_KEY, '"{:"""'); + + expect(getJsonItem(LS_KEY)).toBe(null); + expect(consoleSpy).toHaveBeenCalled(); + + localStorage.setItem(LS_KEY, 'asdas'); + + expect(getJsonItem(LS_KEY)).toBe(null); + expect(consoleSpy).toHaveBeenCalled(); + }); +}); + +describe('updateJsonItem', () => { + afterEach(() => { + localStorage.clear(); + }); + + it('should set data to empty localStorage', () => { + updateJsonItem>(LS_KEY, {}); + + expect(localStorage.getItem(LS_KEY)).toBe(JSON.stringify({})); + }); + + it('should update object in localStorage', () => { + localStorage.setItem(LS_KEY, JSON.stringify({ x: 1 })); + updateJsonItem(LS_KEY, { y: 1 }); + + expect(localStorage.getItem(LS_KEY)).toBe(JSON.stringify({ x: 1, y: 1 })); + }); + + it('should update array in localStorage', () => { + localStorage.setItem(LS_KEY, JSON.stringify([1, 2, 3])); + updateJsonItem(LS_KEY, [4, 5, 6]); + + expect(localStorage.getItem(LS_KEY)).toBe(JSON.stringify([1, 2, 3, 4, 5, 6])); + }); + + it('should update data in localStorage with merge', () => { + localStorage.setItem(LS_KEY, JSON.stringify([3, 4, 5])); + updateJsonItem(LS_KEY, data => [1, 2, ...data]); + + expect(localStorage.getItem(LS_KEY)).toBe(JSON.stringify([1, 2, 3, 4, 5])); + }); +}); diff --git a/frontend/app/common/local-storage.ts b/frontend/app/common/local-storage.ts index 571ed9d7..18e39b28 100644 --- a/frontend/app/common/local-storage.ts +++ b/frontend/app/common/local-storage.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { IS_STORAGE_AVAILABLE } from './constants'; const failMessage = 'remark42: localStorage access denied, check browser preferences'; @@ -20,3 +21,51 @@ export const removeItem = IS_STORAGE_AVAILABLE : () => { console.error(failMessage); // eslint-disable-line no-console }; + +export function getJsonItem(key: string): T | null { + try { + const json = getItem(key); + + if (json === null) { + return null; + } + + const data = JSON.parse(json); + + return data; + } catch (e) { + console.error(`remark42: error on read JSON from ${key} in localStorage`, e); // eslint-disable-line no-console + return null; + } +} + +export function setJsonItem(key: string, data: T) { + try { + setItem(key, JSON.stringify(data)); + } catch (e) { + console.error(`remark42: error on parse JSON from ${key} in localStorage`, e); // eslint-disable-line no-console + } +} + +export function updateJsonItem | any[]>(key: string, value: (data: T) => T): void; +export function updateJsonItem(key: string, value: T): void; +export function updateJsonItem>(key: string, value: T) { + const savedData = getJsonItem(key); + + if (Array.isArray(value)) { + setJsonItem(key, [...savedData, ...value]); + return; + } + + if (value !== null && typeof value === 'object') { + setJsonItem(key, { ...savedData, ...value }); + return; + } + + if (typeof value === 'function') { + setJsonItem(key, value(savedData)); + return; + } + + throw new Error(`remark42: error on update JSON for ${key} in localStorage`); +} diff --git a/frontend/app/components/comment-form/comment-form.test.tsx b/frontend/app/components/comment-form/comment-form.test.tsx index 7848f85c..77f3a1ad 100644 --- a/frontend/app/components/comment-form/comment-form.test.tsx +++ b/frontend/app/components/comment-form/comment-form.test.tsx @@ -4,9 +4,20 @@ import { shallow } from 'enzyme'; import { user } from '@app/testUtils/mocks/user'; import { StaticStore } from '@app/common/static_store'; +import { LS_SAVED_COMMENT_VALUE } from '@app/common/constants'; +import * as localStorageModule from '@app/common/local-storage'; import { CommentForm, Props } from './comment-form'; import { SubscribeByEmail } from './__subscribe-by-email'; +import TextareaAutosize from './textarea-autosize'; + +function createEvent(type: string, value: T) { + const event = new Event(type); + + Object.defineProperty(event, 'target', { value }); + + return event; +} const DEFAULT_PROPS: Readonly> = { mode: 'main', @@ -50,4 +61,74 @@ describe('', () => { expect(wrapper.exists(SubscribeByEmail)).toEqual(false); }); + + describe('initial value of comment', () => { + afterEach(() => { + localStorage.clear(); + }); + it('should has empty value', () => { + localStorage.setItem(LS_SAVED_COMMENT_VALUE, JSON.stringify({ 2: 'text' })); + + const props = { ...DEFAULT_PROPS, user, intl }; + const wrapper = shallow(); + + expect(wrapper.state('text')).toBe(''); + expect(wrapper.find(TextareaAutosize).prop('value')).toBe(''); + }); + + it('should get initial value from localStorage', () => { + const COMMENT_VALUE = 'text'; + + localStorage.setItem(LS_SAVED_COMMENT_VALUE, JSON.stringify({ 1: COMMENT_VALUE })); + + const props = { ...DEFAULT_PROPS, user, intl }; + const wrapper = shallow(); + + expect(wrapper.state('text')).toBe(COMMENT_VALUE); + expect(wrapper.find(TextareaAutosize).prop('value')).toBe(COMMENT_VALUE); + }); + + it('should get initial value from props instead localStorage', () => { + const COMMENT_VALUE = 'text from props'; + + localStorage.setItem(LS_SAVED_COMMENT_VALUE, JSON.stringify({ 1: 'text from localStorage' })); + + const props = { ...DEFAULT_PROPS, user, intl, value: COMMENT_VALUE }; + const wrapper = shallow(); + + expect(wrapper.state('text')).toBe(COMMENT_VALUE); + expect(wrapper.find(TextareaAutosize).prop('value')).toBe(COMMENT_VALUE); + }); + }); + + describe('update value of comment in localStorage', () => { + afterEach(() => { + localStorage.clear(); + }); + it('should update value', () => { + const props = { ...DEFAULT_PROPS, user, intl }; + const wrapper = shallow(); + // @ts-ignore + const instance: CommentForm = wrapper.instance(); + + instance.onInput(createEvent('input', { value: '1' })); + expect(localStorage.getItem(LS_SAVED_COMMENT_VALUE)).toBe('{"1":"1"}'); + + instance.onInput(createEvent('input', { value: '11' })); + expect(localStorage.getItem(LS_SAVED_COMMENT_VALUE)).toBe('{"1":"11"}'); + }); + + it('should clear value after send', async () => { + localStorage.setItem(LS_SAVED_COMMENT_VALUE, JSON.stringify({ '1': 'asd' })); + const updateJsonItemSpy = jest.spyOn(localStorageModule, 'updateJsonItem'); + const props = { ...DEFAULT_PROPS, user, intl }; + const wrapper = shallow(); + // @ts-ignore + const instance: CommentForm = wrapper.instance(); + + await instance.send(createEvent('send', { preventDefault: () => undefined })); + expect(updateJsonItemSpy).toHaveBeenCalled(); + expect(localStorage.getItem(LS_SAVED_COMMENT_VALUE)).toBe(JSON.stringify({})); + }); + }); }); diff --git a/frontend/app/components/comment-form/comment-form.tsx b/frontend/app/components/comment-form/comment-form.tsx index 4f99036b..21334c5e 100644 --- a/frontend/app/components/comment-form/comment-form.tsx +++ b/frontend/app/components/comment-form/comment-form.tsx @@ -11,7 +11,7 @@ import { sleep } from '@app/utils/sleep'; import { replaceSelection } from '@app/utils/replaceSelection'; import { Button } from '@app/components/button'; import Auth from '@app/components/auth'; -import { getItem, setItem } from '@app/common/local-storage'; +import { getJsonItem, updateJsonItem } from '@app/common/local-storage'; import { LS_SAVED_COMMENT_VALUE } from '@app/common/constants'; import { SubscribeByEmail } from './__subscribe-by-email'; @@ -42,7 +42,7 @@ export interface Props { intl: IntlShape; } -interface State { +export interface State { preview: string | null; isErrorShown: boolean; /** error message, if contains newlines, it will be splitted to multiple errors */ @@ -104,13 +104,16 @@ export class CommentForm extends Component { textareaId = textareaId + 1; this.textareaId = `textarea_${textareaId}`; - const savedCommentsJSON = getItem(LS_SAVED_COMMENT_VALUE); - let savedValue = ''; - try { - if (typeof savedCommentsJSON === 'string') { - savedValue = JSON.parse(savedCommentsJSON)[this.props.id] || ''; - } - } catch (e) {} + const savedComments = getJsonItem(LS_SAVED_COMMENT_VALUE); + let text = ''; + + if (savedComments !== null && savedComments[props.id]) { + text = savedComments[props.id]; + } + + if (props.value) { + text = props.value; + } this.state = { preview: null, @@ -119,13 +122,11 @@ export class CommentForm extends Component { errorLock: false, isDisabled: false, maxLength: StaticStore.config.max_comment_size, - text: props.value || savedValue, + text, buttonText: null, }; - this.send = this.send.bind(this); this.getPreview = this.getPreview.bind(this); - this.onInput = this.onInput.bind(this); this.onKeyDown = this.onKeyDown.bind(this); this.onDragOver = this.onDragOver.bind(this); this.onDrop = this.onDrop.bind(this); @@ -169,12 +170,10 @@ export class CommentForm extends Component { } } - onInput(e: Event) { + onInput = (e: Event) => { const { value } = e.target as HTMLInputElement; - try { - setItem(LS_SAVED_COMMENT_VALUE, JSON.stringify({ [this.props.id]: value })); - } catch (e) {} + updateJsonItem(LS_SAVED_COMMENT_VALUE, { [this.props.id]: value }); if (this.state.errorLock) { this.setState({ @@ -183,13 +182,14 @@ export class CommentForm extends Component { }); return; } + this.setState({ isErrorShown: false, errorMessage: null, preview: null, text: value, }); - } + }; async onPaste(e: ClipboardEvent) { if (!(e.clipboardData && e.clipboardData.files.length > 0)) { @@ -200,32 +200,35 @@ export class CommentForm extends Component { await this.uploadImages(files); } - send(e: Event) { - const text = this.textAreaRef.current ? this.textAreaRef.current.getValue() : this.state.text; - const props = this.props; + send = async (e: Event) => { + const { text } = this.state; if (e) e.preventDefault(); if (!text || !text.trim()) return; - if (text === this.props.value) { this.props.onCancel && this.props.onCancel(); this.setState({ preview: null, text: '' }); } this.setState({ isDisabled: true, isErrorShown: false, text }); + try { + await this.props.onSubmit(text, pageTitle || document.title); + updateJsonItem>(LS_SAVED_COMMENT_VALUE, data => { + delete data[this.props.id]; - props - .onSubmit(text, pageTitle || document.title) - .then(() => { - this.setState({ preview: null, text: '' }); - }) - .catch(e => { - const errorMessage = extractErrorMessageFromResponse(e, this.props.intl); - this.setState({ isErrorShown: true, errorMessage }); - }) - .finally(() => this.setState({ isDisabled: false })); - } + return data; + }); + this.setState({ preview: null, text: '' }); + } catch (e) { + this.setState({ + isErrorShown: true, + errorMessage: extractErrorMessageFromResponse(e, this.props.intl), + }); + } + + this.setState({ isDisabled: false }); + }; getPreview() { const text = this.textAreaRef.current ? this.textAreaRef.current.getValue() : this.state.text; diff --git a/frontend/app/components/comment/comment.tsx b/frontend/app/components/comment/comment.tsx index 36865c9a..b1528f7c 100644 --- a/frontend/app/components/comment/comment.tsx +++ b/frontend/app/components/comment/comment.tsx @@ -835,7 +835,6 @@ class Comment extends Component { intl={this.props.intl} user={props.user} theme={props.theme} - value="" mode="reply" mix="comment__input" onSubmit={(text, title) => this.addComment(text, title, o.id)}