test 123
\n\n\n", comment.Text) assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig) - assert.Equal(t, store.User{Name: "developer one", ID: "dev", - Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"}, + assert.Equal(t, store.User{Name: "developer one", ID: "dev", Admin: true, Blocked: false, + IP: "dbc7c999343f003f189f70aaf52cc04443f90790"}, comment.User) t.Logf("%+v", comment) } @@ -336,7 +336,7 @@ func TestRest_UserAllData(t *testing.T) { ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) assert.True(t, strings.HasPrefix(string(ungzBody), - `{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`)) + `{"info": {"name":"developer one","id":"dev","picture":"","admin":true}, "comments":[{`)) assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside") t.Logf("%s", string(ungzBody)) @@ -347,7 +347,7 @@ func TestRest_UserAllData(t *testing.T) { err = json.Unmarshal(ungzBody, &parsed) assert.Nil(t, err) - assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true}, parsed.Info) + assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "", Admin: true}, parsed.Info) assert.Equal(t, 3, len(parsed.Comments)) req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil) @@ -387,7 +387,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) { ungzBody, err := ioutil.ReadAll(ungzReader) assert.NoError(t, err) assert.True(t, strings.HasPrefix(string(ungzBody), - `{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`)) + `{"info": {"name":"developer one","id":"dev","picture":"","admin":true}, "comments":[{`)) assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside") } diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index 042ba13f..9f40a7f3 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -289,8 +289,7 @@ func TestRest_UserInfo(t *testing.T) { user := store.User{} err := json.Unmarshal([]byte(body), &user) assert.Nil(t, err) - assert.Equal(t, store.User{Name: "developer one", ID: "dev", - Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: ""}, user) + assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "", Admin: true, Blocked: false, IP: ""}, user) } func TestRest_Count(t *testing.T) { diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index 476b08f5..7ed3be28 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -12,8 +12,10 @@ import ( "testing" "time" - "github.com/coreos/bbolt" + bolt "github.com/coreos/bbolt" "github.com/go-pkgz/auth" + "github.com/go-pkgz/auth/avatar" + "github.com/go-pkgz/auth/token" R "github.com/go-pkgz/rest" "github.com/go-pkgz/rest/cache" "github.com/stretchr/testify/assert" @@ -23,7 +25,6 @@ import ( "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/avatar" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) @@ -61,8 +62,7 @@ func TestRest_GetStarted(t *testing.T) { } func TestRest_Shutdown(t *testing.T) { - srv := Rest{Authenticator: auth.Service{}, AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), - RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}} + srv := Rest{Authenticator: auth.Service{}, ImageProxy: &proxy.Image{}} go func() { time.Sleep(100 * time.Millisecond) @@ -91,11 +91,11 @@ func TestRest_filterComments(t *testing.T) { func TestRest_RunStaticSSLMode(t *testing.T) { srv := Rest{ - Authenticator: auth.Service{}, - AvatarProxy: &proxy.Avatar{ - Store: avatar.NewLocalFS("/tmp", 300), - RoutePath: "/api/v1/avatar", - }, + Authenticator: *auth.NewService(auth.Opts{ + AvatarStore: avatar.NewLocalFS("/tmp"), + AvatarResizeLimit: 300, + }), + ImageProxy: &proxy.Image{}, SSLConfig: SSLConfig{ SSLMode: Static, @@ -144,11 +144,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) { func TestRest_RunAutocertModeHTTPOnly(t *testing.T) { srv := Rest{ Authenticator: auth.Service{}, - AvatarProxy: &proxy.Avatar{ - Store: avatar.NewLocalFS("/tmp", 300), - RoutePath: "/api/v1/avatar", - }, - ImageProxy: &proxy.Image{}, + ImageProxy: &proxy.Image{}, SSLConfig: SSLConfig{ SSLMode: Auto, Port: 8443, @@ -193,17 +189,18 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) { MaxVotes: service.UnlimitedVotes, } - //DevPasswd: "password", - // Providers: nil, - // KeyStore: adminStore, - // JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour), srv = &Rest{ - DataService: dataStore, - Authenticator: *auth.NewService(auth.Opts{}), - Cache: &cache.Nop{}, - WebRoot: "/tmp", - RemarkURL: "https://demo.remark42.com", - AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"}, + DataService: dataStore, + Authenticator: *auth.NewService(auth.Opts{ + DevPasswd: "password", + SecretReader: token.SecretFunc(func(id string) (string, error) { return "secret", nil }), + AvatarStore: avatar.NewLocalFS("/tmp"), + AvatarResizeLimit: 300, + }), + Cache: &cache.Nop{}, + WebRoot: "/tmp", + RemarkURL: "https://demo.remark42.com", + ImageProxy: &proxy.Image{}, ReadOnlyAge: 10, CommentFormatter: store.NewCommentFormatter(&proxy.Image{}), @@ -257,21 +254,21 @@ func post(t *testing.T, url string, body string) (*http.Response, error) { func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string { b, err := json.Marshal(c) - assert.Nil(t, err, "can't marshal comment %+v", c) + require.Nil(t, err, "can't marshal comment %+v", c) client := &http.Client{Timeout: 5 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b)) - assert.Nil(t, err) + require.Nil(t, err) req.SetBasicAuth("dev", "password") resp, err := client.Do(req) - assert.Nil(t, err) - assert.Equal(t, http.StatusCreated, resp.StatusCode) + require.Nil(t, err) + require.Equal(t, http.StatusCreated, resp.StatusCode) b, err = ioutil.ReadAll(resp.Body) - assert.Nil(t, err) + require.Nil(t, err) crResp := R.JSON{} err = json.Unmarshal(b, &crResp) - assert.Nil(t, err) + require.Nil(t, err) time.Sleep(time.Nanosecond * 10) return crResp["id"].(string) } diff --git a/backend/app/rest/proxy/avatar.go b/backend/app/rest/proxy/avatar.go deleted file mode 100644 index 7b71bcf1..00000000 --- a/backend/app/rest/proxy/avatar.go +++ /dev/null @@ -1,118 +0,0 @@ -package proxy - -import ( - "io" - "log" - "net/http" - "strconv" - "strings" - "time" - - "github.com/go-chi/chi" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/rest" - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/avatar" -) - -// Avatar provides http handler for avatars from avatar.Store -// On user login auth will call Put and it will retrieve and save picture locally. -type Avatar struct { - Store avatar.Store - RoutePath string - RemarkURL string -} - -// Put stores retrieved avatar to avatar.Store. Gets image from user info. Returns proxied url -func (p *Avatar) Put(u store.User) (avatarURL string, err error) { - - // no picture for user, try default avatar - if u.Picture == "" { - return "", errors.Errorf("no picture for %s", u.ID) - } - - // load avatar from remote location - client := http.Client{Timeout: 10 * time.Second} - var resp *http.Response - err = retry(5, time.Second, func() error { - var e error - resp, e = client.Get(u.Picture) - return e - }) - if err != nil { - return "", errors.Wrap(err, "failed to fetch avatar from the orig") - } - - defer func() { - if e := resp.Body.Close(); e != nil { - log.Printf("[WARN] can't close response body, %s", e) - } - }() - - if resp.StatusCode != http.StatusOK { - return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status) - } - - avatarID, err := p.Store.Put(u.ID, resp.Body) // put returns avatar base name, like 123456.image - if err != nil { - return "", err - } - - log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatarID, u.Name) - return p.RemarkURL + p.RoutePath + "/" + avatarID, nil -} - -// Routes returns auth routes for given provider -func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string, chi.Router) { - router := chi.NewRouter() - router.Use(middlewares...) - - // GET /123456789.image - router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) { - - avatarID := chi.URLParam(r, "avatar") - - // enforce client-side caching - etag := `"` + p.Store.ID(avatarID) + `"` - w.Header().Set("Etag", etag) - w.Header().Set("Cache-Control", "max-age=604800") // 7 days - if match := r.Header.Get("If-None-Match"); match != "" { - if strings.Contains(match, etag) { - w.WriteHeader(http.StatusNotModified) - return - } - } - - avReader, size, err := p.Store.Get(avatarID) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar") - return - } - - defer func() { - if e := avReader.Close(); e != nil { - log.Printf("[WARN] can't close avatar reader for %s, %s", avatarID, e) - } - }() - - w.Header().Set("Content-Type", "image/*") - w.Header().Set("Content-Length", strconv.Itoa(size)) - w.WriteHeader(http.StatusOK) - if _, err = io.Copy(w, avReader); err != nil { - log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err) - } - }) - - return p.RoutePath, router -} - -func retry(retries int, delay time.Duration, fn func() error) (err error) { - for i := 0; i < retries; i++ { - if err = fn(); err == nil { - return nil - } - time.Sleep(delay) - } - return errors.Wrap(err, "retry failed") -} diff --git a/backend/app/rest/proxy/avatar_test.go b/backend/app/rest/proxy/avatar_test.go deleted file mode 100644 index 53eba9a6..00000000 --- a/backend/app/rest/proxy/avatar_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package proxy - -import ( - "bytes" - "errors" - "fmt" - "io" - "log" - "net/http" - "net/http/httptest" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/avatar" -) - -func TestAvatar_Put(t *testing.T) { - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/pic.png" { - w.Header().Set("Content-Type", "image/*") - fmt.Fprint(w, "some picture bin data") - return - } - http.Error(w, "not found", http.StatusNotFound) - })) - defer ts.Close() - - p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)} - os.MkdirAll("/tmp/avatars.test", 0700) - defer os.RemoveAll("/tmp/avatars.test") - - u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"} - res, err := p.Put(u) - assert.NoError(t, err) - assert.Equal(t, "http://localhost:8080/avatar/b3daa77b4c04a9551b8781d03191fe098f325e67.image", res) - fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image") - assert.NoError(t, err) - assert.Equal(t, int64(21), fi.Size()) - - u.ID = "user2" - res, err = p.Put(u) - assert.NoError(t, err) - assert.Equal(t, "http://localhost:8080/avatar/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", res) - fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image") - assert.NoError(t, err) - assert.Equal(t, int64(21), fi.Size()) -} - -func TestAvatar_PutFailed(t *testing.T) { - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - log.Print("request: ", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - })) - defer ts.Close() - - p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)} - - u := store.User{ID: "user1", Name: "user1 name"} - _, err := p.Put(u) - assert.EqualError(t, err, "no picture for user1") - - u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"} - _, err = p.Put(u) - require.Error(t, err) - assert.Contains(t, err.Error(), "connect: connection refused") - - u = store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/avatar/pic"} - _, err = p.Put(u) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get avatar from the orig") -} - -func TestAvatar_Routes(t *testing.T) { - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/pic.png" { - w.Header().Set("Content-Type", "image/*") - w.Header().Set("Custom-Header", "xyz") - fmt.Fprint(w, "some picture bin data") - return - } - http.Error(w, "not found", http.StatusNotFound) - })) - defer ts.Close() - - p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)} - os.MkdirAll("/tmp/avatars.test", 0700) - defer os.RemoveAll("/tmp/avatars.test") - - u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"} - _, err := p.Put(u) - assert.NoError(t, err) - - // status 400 - req, err := http.NewRequest("GET", "/some_random_name.image", nil) - if err != nil { - t.Fatal(err) - } - - rr := httptest.NewRecorder() - _, routes := p.Routes() - handler := http.Handler(routes) - handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusBadRequest, rr.Code) - - // status 200 - req, err = http.NewRequest("GET", "/b3daa77b4c04a9551b8781d03191fe098f325e67.image", nil) - if err != nil { - t.Fatal(err) - } - - rr = httptest.NewRecorder() - _, routes = p.Routes() - handler = http.Handler(routes) - handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusOK, rr.Code) - - assert.Equal(t, []string{"image/*"}, rr.HeaderMap["Content-Type"]) - assert.Equal(t, []string{"21"}, rr.HeaderMap["Content-Length"]) - assert.Equal(t, []string(nil), rr.HeaderMap["Custom-Header"], "strip all custom headers") - assert.NotNil(t, rr.HeaderMap["Etag"]) - - bb := bytes.Buffer{} - sz, err := io.Copy(&bb, rr.Body) - assert.NoError(t, err) - assert.Equal(t, int64(21), sz) - assert.Equal(t, "some picture bin data", bb.String()) - - // status 304 - req, err = http.NewRequest("GET", "/some_random_name.image", nil) - if err != nil { - t.Fatal(err) - } - req.Header.Add("If-None-Match", `"a008de0a2ccb3308b5d99ffff66436e15538f701"`) // hash of `some_random_name.image` since the file doesn't exist - - rr = httptest.NewRecorder() - _, routes = p.Routes() - handler = http.Handler(routes) - handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusNotModified, rr.Code) - assert.Equal(t, []string{`"a008de0a2ccb3308b5d99ffff66436e15538f701"`}, rr.HeaderMap["Etag"]) -} - -func TestAvatar_Retry(t *testing.T) { - i := 0 - err := retry(5, time.Millisecond, func() error { - if i == 3 { - return nil - } - i++ - return errors.New("err") - }) - assert.Nil(t, err) - assert.Equal(t, 3, i) - - st := time.Now() - err = retry(5, time.Millisecond, func() error { - return errors.New("err") - }) - assert.NotNil(t, err) - assert.True(t, time.Since(st) >= time.Microsecond*5) -} diff --git a/backend/app/rest/proxy/image.go b/backend/app/rest/proxy/image.go index 284c8e13..4d33311d 100644 --- a/backend/app/rest/proxy/image.go +++ b/backend/app/rest/proxy/image.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "git.tkginternal.com/commons/pkg/repeater" + "github.com/PuerkitoBio/goquery" "github.com/go-chi/chi" "github.com/pkg/errors" @@ -52,7 +54,7 @@ func (p Image) Routes() chi.Router { client := http.Client{Timeout: 30 * time.Second} var resp *http.Response - err = retry(5, time.Second, func() error { + err = repeater.NewDefault(5, time.Second).Do(func() error { var e error resp, e = client.Get(string(src)) return e diff --git a/backend/app/rest/user.go b/backend/app/rest/user.go index 98d0092c..7727e137 100644 --- a/backend/app/rest/user.go +++ b/backend/app/rest/user.go @@ -1,15 +1,14 @@ package rest import ( - "context" - "errors" "net/http" + "github.com/go-pkgz/auth/token" + "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store" ) -type contextKey string - // MustGetUserInfo fails if can't extract user data from the request. // should be called from authed controllers only func MustGetUserInfo(r *http.Request) store.User { @@ -23,20 +22,36 @@ func MustGetUserInfo(r *http.Request) store.User { // GetUserInfo returns user from request context func GetUserInfo(r *http.Request) (user store.User, err error) { - ctx := r.Context() - if ctx == nil { - return store.User{}, errors.New("no info about user") - } - if u, ok := ctx.Value(contextKey("user")).(store.User); ok { - return u, nil + u, err := token.GetUserInfo(r) + if err != nil { + return store.User{}, errors.Wrap(err, "can't extract user info from the token") } - return store.User{}, errors.New("user can't be parsed") + return store.User{ + Name: u.Name, + ID: u.ID, + IP: u.IP, + Picture: u.Picture, + Admin: u.IsAdmin(), + Verified: u.BoolAttr("verified"), + Blocked: u.BoolAttr("blocked"), + }, nil + } // SetUserInfo sets user into request context func SetUserInfo(r *http.Request, user store.User) *http.Request { - ctx := r.Context() - ctx = context.WithValue(ctx, contextKey("user"), user) - return r.WithContext(ctx) + u := token.User{ + ID: user.ID, + Name: user.Name, + Picture: user.Picture, + IP: user.IP, + Attributes: map[string]interface{}{ + "blocked": user.Blocked, + "verified": user.Verified, + }, + } + u.SetAdmin(user.Admin) + + return token.SetUserInfo(r, u) } diff --git a/backend/app/store/avatar/bolt.go b/backend/app/store/avatar/bolt.go deleted file mode 100644 index 826a1ded..00000000 --- a/backend/app/store/avatar/bolt.go +++ /dev/null @@ -1,144 +0,0 @@ -package avatar - -import ( - "bytes" - "crypto/sha1" - "encoding/hex" - "io" - "io/ioutil" - "log" - - "github.com/coreos/bbolt" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// BoltDB implements avatar store with bolt -// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket -// to keep sha1 of picture. avatarID (base file name) used as a key for both. -type BoltDB struct { - fileName string // full path to boltdb - resizeLimit int - db *bolt.DB -} - -const avatarsBktName = "avatars" -const metasBktName = "metas" - -// NewBoltDB makes bolt avatar store -func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, error) { - db, err := bolt.Open(fileName, 0600, &options) - if err != nil { - return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName) - } - err = db.Update(func(tx *bolt.Tx) error { - if _, e := tx.CreateBucketIfNotExists([]byte(avatarsBktName)); e != nil { - return errors.Wrapf(e, "failed to create top level bucket %s", avatarsBktName) - } - _, e := tx.CreateBucketIfNotExists([]byte(metasBktName)) - return errors.Wrapf(e, "failed to create top metas bucket %s", metasBktName) - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName) - } - return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil -} - -// Put avatar to bolt, key by avatarID. Trying to resize image and lso calculates sha1 of the file for ID func -func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) { - id := encodeID(userID) - - // Trying to resize avatar. - if reader = resize(reader, b.resizeLimit); reader == nil { - return "", errors.New("avatar resize reader is nil") - } - - avatarID := id + imgSfx - err = b.db.Update(func(tx *bolt.Tx) error { - buf := &bytes.Buffer{} - if _, err = io.Copy(buf, reader); err != nil { - return errors.Wrapf(err, "can't read avatar %s", avatarID) - } - - if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil { - return errors.Wrapf(err, "can't put to bucket with %s", avatarID) - } - // store sha1 of the image - return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID))) - }) - return avatarID, err -} - -// Get avatar reader for avatar id.image, avatarID used as the direct key -func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error) { - buf := &bytes.Buffer{} - err = b.db.View(func(tx *bolt.Tx) error { - data := tx.Bucket([]byte(avatarsBktName)).Get([]byte(avatarID)) - if data == nil { - return errors.Errorf("can't load avatar %s", avatarID) - } - size, err = buf.Write(data) - return errors.Wrapf(err, "failed to write for %s", avatarID) - }) - return ioutil.NopCloser(buf), size, err -} - -// ID returns a fingerprint of the avatar content. -func (b *BoltDB) ID(avatarID string) (id string) { - data := []byte{} - err := b.db.View(func(tx *bolt.Tx) error { - if data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)); data == nil { - return errors.Errorf("can't load avatar's id for %s", avatarID) - } - return nil - }) - - if err != nil { // failed to get ID, use encoded avatarID - log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err) - return store.EncodeID(avatarID) - } - - return string(data) -} - -// Remove avatar from bolt -func (b *BoltDB) Remove(avatarID string) (err error) { - return b.db.Update(func(tx *bolt.Tx) error { - bkt := tx.Bucket([]byte(avatarsBktName)) - if bkt.Get([]byte(avatarID)) == nil { - return errors.Errorf("avatar key not found, %s", avatarID) - } - if err = tx.Bucket([]byte(avatarsBktName)).Delete([]byte(avatarID)); err != nil { - return errors.Wrapf(err, "can't delete avatar object %s", avatarID) - } - return errors.Wrapf(tx.Bucket([]byte(metasBktName)).Delete([]byte(avatarID)), - "can't delete meta object %s", avatarID) - }) -} - -// List all avatars (ids) from metas bucket -// note: id includes .image suffix -func (b *BoltDB) List() (ids []string, err error) { - err = b.db.View(func(tx *bolt.Tx) error { - return tx.Bucket([]byte(metasBktName)).ForEach(func(k, _ []byte) error { - ids = append(ids, string(k)) - return nil - }) - }) - return ids, errors.Wrap(err, "failed to list") -} - -// Close bolt store -func (b *BoltDB) Close() error { - return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName) -} - -func (b *BoltDB) sha1(data []byte, avatarID string) (id string) { - h := sha1.New() - if _, err := h.Write(data); err != nil { - log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err) - return store.EncodeID(avatarID) - } - return hex.EncodeToString(h.Sum(nil)) -} diff --git a/backend/app/store/avatar/bolt_test.go b/backend/app/store/avatar/bolt_test.go deleted file mode 100644 index b9ddac23..00000000 --- a/backend/app/store/avatar/bolt_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package avatar - -import ( - "io/ioutil" - "os" - "sort" - "strings" - "testing" - - "github.com/coreos/bbolt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var testDb = "/tmp/test-remark-avatars.db" - -func TestBoltDB_PutAndGet(t *testing.T) { - var b Store = prepBoltStore(t) - defer func() { - assert.Nil(t, b.Close()) - os.Remove(testDb) - }() - - avatar, err := b.Put("user1", strings.NewReader("some picture bin data")) - require.Nil(t, err) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) - - rd, size, err := b.Get(avatar) - require.Nil(t, err) - assert.Equal(t, 21, size) - data, err := ioutil.ReadAll(rd) - require.Nil(t, err) - assert.Equal(t, "some picture bin data", string(data)) - - _, _, err = b.Get("bad avatar") - assert.NotNil(t, err) - - // check IDs - assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar)) - assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encoded avatar id") - - l, err := b.List() - require.Nil(t, err) - assert.Equal(t, 1, len(l)) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0]) -} - -func TestBoltDB_Remove(t *testing.T) { - b := prepBoltStore(t) - defer func() { - assert.Nil(t, b.Close()) - os.Remove(testDb) - }() - - assert.NotNil(t, b.Remove("no-such-thing.image")) - - avatar, err := b.Put("user1", strings.NewReader("some picture bin data")) - require.Nil(t, err) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) - assert.NoError(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one") - assert.NotNil(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") -} - -func TestBoltDB_List(t *testing.T) { - b := prepBoltStore(t) - defer func() { - assert.Nil(t, b.Close()) - os.Remove(testDb) - }() - - // write some avatars - _, err := b.Put("user1", strings.NewReader("some picture bin data 1")) - require.Nil(t, err) - _, err = b.Put("user2", strings.NewReader("some picture bin data 2")) - require.Nil(t, err) - _, err = b.Put("user3", strings.NewReader("some picture bin data 3")) - require.Nil(t, err) - - l, err := b.List() - assert.NoError(t, err) - assert.Equal(t, 3, len(l), "3 avatars listed") - sort.Strings(l) - assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) - - r, size, err := b.Get("0b7f849446d3383546d15a480966084442cd2193.image") - assert.Nil(t, err) - assert.Equal(t, 23, size) - data, err := ioutil.ReadAll(r) - assert.Nil(t, err) - assert.Equal(t, "some picture bin data 3", string(data)) -} - -// makes new boltdb, put two records -func prepBoltStore(t *testing.T) *BoltDB { - os.Remove(testDb) - boltStore, err := NewBoltDB(testDb, bolt.Options{}, 0) - require.Nil(t, err) - return boltStore -} diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go deleted file mode 100644 index 6a20253e..00000000 --- a/backend/app/store/avatar/gridfs.go +++ /dev/null @@ -1,125 +0,0 @@ -package avatar - -import ( - "bytes" - "io" - "io/ioutil" - "log" - "time" - - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// NewGridFS makes gridfs (mongo) avatar store -func NewGridFS(conn *mongo.Connection, resizeLimit int) *GridFS { - return &GridFS{Connection: conn, resizeLimit: resizeLimit} -} - -// GridFS implements Store for GridFS -type GridFS struct { - Connection *mongo.Connection - resizeLimit int -} - -// Put avatar to gridfs object, try to resize -func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := encodeID(userID) - err = gf.Connection.WithDB(func(dbase *mgo.Database) error { - fh, e := dbase.GridFS("fs").Create(id + imgSfx) - if e != nil { - return e - } - defer func() { - if err = fh.Close(); err != nil { - log.Printf("[WARN] can't close avatar file %v, %s", fh, err) - } - }() - - // Trying to resize avatar. - if reader = resize(reader, gf.resizeLimit); reader == nil { - return errors.New("avatar resize reader is nil") - } - _, e = io.Copy(fh, reader) - return e - }) - return id + imgSfx, err -} - -// Get avatar reader for avatar id.image -func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) { - buf := &bytes.Buffer{} - err = gf.Connection.WithDB(func(dbase *mgo.Database) error { - fh, e := dbase.GridFS("fs").Open(avatar) - if e != nil { - return errors.Wrapf(e, "can't load avatar %s", avatar) - } - if _, e = io.Copy(buf, fh); e != nil { - return errors.Wrapf(e, "can't copy avatar %s", avatar) - } - size = int(fh.Size()) - return fh.Close() - }) - return ioutil.NopCloser(buf), size, err -} - -// ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly -func (gf *GridFS) ID(avatar string) (id string) { - err := gf.Connection.WithDB(func(dbase *mgo.Database) error { - fh, e := dbase.GridFS("fs").Open(avatar) - if e != nil { - return errors.Wrapf(e, "can't open avatar %s", avatar) - } - id = fh.MD5() - return errors.Wrapf(fh.Close(), "can't close avatar") - }) - if err != nil { - log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err) - return store.EncodeID(avatar) - } - return id -} - -// Remove avatar from gridfs -func (gf *GridFS) Remove(avatar string) error { - return gf.Connection.WithDB(func(dbase *mgo.Database) error { - fh, e := dbase.GridFS("fs").Open(avatar) - if e != nil { - return errors.Wrapf(e, "can't get avatar %s", avatar) - } - if e = fh.Close(); e != nil { - log.Printf("[WARN] can't close avatar %s, %s", avatar, e) - } - return dbase.GridFS("fs").Remove(avatar) - }) -} - -// List all avatars (ids) on gfs -// note: id includes .image suffix -func (gf *GridFS) List() (ids []string, err error) { - - type gfsFile struct { - UploadDate time.Time `bson:"uploadDate"` - Length int64 `bson:",minsize"` - MD5 string - Filename string `bson:",omitempty"` - } - - files := []gfsFile{} - err = gf.Connection.WithDB(func(dbase *mgo.Database) error { - return dbase.GridFS("fs").Find(nil).All(&files) - }) - - for _, f := range files { - ids = append(ids, f.Filename) - } - return ids, errors.Wrap(err, "can't list avatars") -} - -// Close gridfs does nothing but satisfies interface -func (gf *GridFS) Close() error { - return nil -} diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go deleted file mode 100644 index 4ca28741..00000000 --- a/backend/app/store/avatar/gridfs_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package avatar - -import ( - "io/ioutil" - "sort" - "strings" - "testing" - - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGridFS_PutAndGet(t *testing.T) { - p, skip := prepGFStore(t) - if skip { - return - } - avatar, err := p.Put("user1", strings.NewReader("some picture bin data")) - require.Nil(t, err) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) - - rd, size, err := p.Get(avatar) - require.Nil(t, err) - assert.Equal(t, 21, size) - data, err := ioutil.ReadAll(rd) - require.Nil(t, err) - assert.Equal(t, "some picture bin data", string(data)) - - _, _, err = p.Get("bad avatar") - assert.NotNil(t, err) - - assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar)) - assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id") - - l, err := p.List() - require.Nil(t, err) - assert.Equal(t, 1, len(l)) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0]) -} - -func TestGridFS_Remove(t *testing.T) { - p, skip := prepGFStore(t) - if skip { - return - } - - assert.NotNil(t, p.Remove("no-such-thing.image")) - - avatar, err := p.Put("user1", strings.NewReader("some picture bin data")) - require.Nil(t, err) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) - assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one") - assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") -} - -func TestGridFS_List(t *testing.T) { - p, skip := prepGFStore(t) - if skip { - return - } - // write some avatars - _, err := p.Put("user1", strings.NewReader("some picture bin data 1")) - require.Nil(t, err) - _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) - require.Nil(t, err) - _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) - require.Nil(t, err) - - l, err := p.List() - assert.NoError(t, err) - assert.Equal(t, 3, len(l), "3 avatars listed") - sort.Strings(l) - assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) - - r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") - assert.Nil(t, err) - assert.Equal(t, 23, size) - data, err := ioutil.ReadAll(r) - assert.Nil(t, err) - assert.Equal(t, "some picture bin data 3", string(data)) -} - -func prepGFStore(t *testing.T) (Store, bool) { - conn, err := mongo.MakeTestConnection(t) - if err != nil { - return nil, true - } - _ = conn.WithCustomCollection("fs.chunks", func(coll *mgo.Collection) error { - return coll.DropCollection() - }) - _ = conn.WithCustomCollection("fs.files", func(coll *mgo.Collection) error { - return coll.DropCollection() - }) - return NewGridFS(conn, 0), false -} diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go deleted file mode 100644 index 9b34ac3d..00000000 --- a/backend/app/store/avatar/localfs.go +++ /dev/null @@ -1,129 +0,0 @@ -package avatar - -import ( - "fmt" - "hash/crc64" - "io" - "log" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// LocalFS implements Store for local file system -type LocalFS struct { - storePath string - resizeLimit int - ctcTable *crc64.Table - once sync.Once -} - -// NewLocalFS makes file-system avatar store -func NewLocalFS(storePath string, resizeLimit int) *LocalFS { - return &LocalFS{storePath: storePath, resizeLimit: resizeLimit} -} - -// Put avatar for userID to file and return avatar's file name (base), like 12345678.image -// userID can be avatarID as well, in this case encoding just strip .image prefix -func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := encodeID(userID) - location := fs.location(id) // location adds partition to path - - if _, err = os.Stat(location); os.IsNotExist(err) { - if e := os.Mkdir(location, 0700); e != nil { - return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location) - } - } - - avFile := path.Join(location, id+imgSfx) - fh, err := os.Create(avFile) - if err != nil { - return "", errors.Wrapf(err, "can't create file %s", avFile) - } - defer func() { - if e := fh.Close(); e != nil { - log.Printf("[WARN] can't close avatar file %s, %s", avFile, e) - } - }() - - // Trying to resize avatar. - if reader = resize(reader, fs.resizeLimit); reader == nil { - return "", errors.New("avatar resize reader is nil") - } - - if _, err = io.Copy(fh, reader); err != nil { - return "", errors.Wrapf(err, "can't save file %s", avFile) - } - return id + imgSfx, nil -} - -// Get avatar reader for avatar id.image -func (fs *LocalFS) Get(avatar string) (reader io.ReadCloser, size int, err error) { - location := fs.location(strings.TrimSuffix(avatar, imgSfx)) - avFile := path.Join(location, avatar) - fh, err := os.Open(avFile) - if err != nil { - return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar) - } - if fi, e := fh.Stat(); e == nil { - size = int(fi.Size()) - } - return fh, size, nil -} - -// ID returns a fingerprint of the avatar content. -func (fs *LocalFS) ID(avatar string) (id string) { - location := fs.location(strings.TrimSuffix(avatar, imgSfx)) - avFile := path.Join(location, avatar) - fi, err := os.Stat(avFile) - if err != nil { - log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err) - return store.EncodeID(avatar) - } - return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10)) -} - -// Remove avatar file -func (fs *LocalFS) Remove(avatar string) error { - location := fs.location(strings.TrimSuffix(avatar, imgSfx)) - avFile := path.Join(location, avatar) - return os.Remove(avFile) -} - -// List all avatars (ids) on local file system -// note: id includes .image suffix -func (fs *LocalFS) List() (ids []string, err error) { - err = filepath.Walk(fs.storePath, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) { - ids = append(ids, info.Name()) - } - return nil - }) - return ids, errors.Wrap(err, "can't list avatars") -} - -// Close gridfs does nothing but satisfies interface -func (fs *LocalFS) Close() error { - return nil -} - -// get location (directory) for user id by adding partition to final path in order to keep files -// in different subdirectories and avoid too many files in a single place. -// the end result is a full path like this - /tmp/avatars.test/92 -func (fs *LocalFS) location(id string) string { - fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) }) - checksum64 := crc64.Checksum([]byte(id), fs.ctcTable) - partition := checksum64 % 100 - return path.Join(fs.storePath, fmt.Sprintf("%02d", partition)) -} diff --git a/backend/app/store/avatar/localfs_test.go b/backend/app/store/avatar/localfs_test.go deleted file mode 100644 index 922c420f..00000000 --- a/backend/app/store/avatar/localfs_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package avatar - -import ( - "io/ioutil" - "os" - "sort" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestAvatarStoreFS_Put(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - avatar, err := p.Put("user1", nil) - assert.Equal(t, "", avatar) - assert.EqualError(t, err, "avatar resize reader is nil") - - avatar, err = p.Put("user1", strings.NewReader("some picture bin data")) - require.Nil(t, err) - assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar) - fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image") - assert.NoError(t, err) - assert.Equal(t, int64(21), fi.Size()) - - avatar, err = p.Put("user2", strings.NewReader("some picture bin data 123")) - require.Nil(t, err) - assert.Equal(t, "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", avatar) - fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image") - assert.NoError(t, err) - assert.Equal(t, int64(25), fi.Size()) - - // with encoded id - avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123")) - require.Nil(t, err) - assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar) - fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image") - assert.NoError(t, err) - assert.Equal(t, int64(25), fi.Size()) - - // with resize - file, e := os.Open("testdata/circles.png") - require.Nil(t, e) - avatar, err = p.Put("user3", file) - require.Nil(t, err) - assert.Equal(t, "0b7f849446d3383546d15a480966084442cd2193.image", avatar) - fi, err = os.Stat("/tmp/avatars.test/60/0b7f849446d3383546d15a480966084442cd2193.image") - assert.NoError(t, err) - assert.Equal(t, int64(6986), fi.Size()) - - p = NewLocalFS("/dev/null", 300) - _, err = p.Put("user1", strings.NewReader("some picture bin data")) - assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory") -} - -func TestAvatarStoreFS_Get(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test/30", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - // file not exists - r, size, err := p.Get("some_random_name.image") - // nil, 0, errors.Wrapf(err, "can't load avatar %s, id") - assert.Nil(t, r) - assert.Equal(t, 0, size) - assert.EqualError(t, err, "can't load avatar some_random_name.image, id: open /tmp/avatars.test/91/some_random_name.image: no such file or directory") - // file exists - err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666) - assert.Nil(t, err) - r, size, err = p.Get("b3daa77b4c04a9551b8781d03191fe098f325e67.image") - - assert.Nil(t, err) - assert.Equal(t, 9, size) - data, err := ioutil.ReadAll(r) - assert.Nil(t, err) - assert.Equal(t, "something", string(data)) -} - -func TestAvatarStoreFS_Location(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - - tbl := []struct { - id string - res string - }{ - {"abc", "/tmp/avatars.test/35"}, - {"xyz", "/tmp/avatars.test/69"}, - {"blah blah", "/tmp/avatars.test/29"}, - {"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"}, - } - - for i, tt := range tbl { - assert.Equal(t, tt.res, p.location(tt.id), "test #%d", i) - } -} - -func TestAvatarStoreFS_ID(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test/30", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - // file not exists - id := p.ID("some_random_name.image") - assert.Equal(t, "a008de0a2ccb3308b5d99ffff66436e15538f701", id) // store.EncodeID("some_random_name.image") - // file exists - err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666) - require.NoError(t, err) - touch := time.Date(2017, 7, 14, 2, 40, 0, 0, time.UTC) // 1500000000 - err = os.Chtimes("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", touch, touch) - require.NoError(t, err) - id = p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image") - assert.Equal(t, "325d5b451f32c2f8e7f30a9fd65bff6a42954d9a", id) // store.EncodeID("b3daa77b4c04a9551b8781d03191fe098f325e67.image1500000000") -} - -func TestAvatarStoreFS_Remove(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test/30", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - assert.NotNil(t, p.Remove("no-such-avatar"), "remove non-existing avatar") - err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666) - require.NoError(t, err) - - assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image")) - _, err = os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image") - assert.NotNil(t, err, "removed for real") - t.Log(err) -} - -func TestAvatarStoreFS_List(t *testing.T) { - p := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - // write some avatars - _, err = p.Put("user1", strings.NewReader("some picture bin data 1")) - require.Nil(t, err) - _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) - require.Nil(t, err) - _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) - require.Nil(t, err) - - l, err := p.List() - assert.NoError(t, err) - assert.Equal(t, 3, len(l), "3 avatars listed") - sort.Strings(l) - assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) - - r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") - assert.Nil(t, err) - assert.Equal(t, 23, size) - data, err := ioutil.ReadAll(r) - assert.Nil(t, err) - assert.Equal(t, "some picture bin data 3", string(data)) -} - -func BenchmarkAvatarStoreFS_ID(b *testing.B) { - p := NewLocalFS("/tmp/avatars.test", 300) - os.MkdirAll("/tmp/avatars.test/30", 0700) - defer os.RemoveAll("/tmp/avatars.test") - err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666) - require.NoError(b, err) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image") - } -} diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go deleted file mode 100644 index eb9a3aed..00000000 --- a/backend/app/store/avatar/store.go +++ /dev/null @@ -1,110 +0,0 @@ -// Package avatar defines store interface and implements local (fs), gridfs (mongo) and boltdb stores. -// -package avatar - -//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go" - -import ( - "bytes" - "image" - "strings" - - // Initializing packages for supporting GIF and JPEG formats. - _ "image/gif" - _ "image/jpeg" - "image/png" - "io" - "log" - "regexp" - - "github.com/umputun/remark/backend/app/store" - "golang.org/x/image/draw" -) - -// imgSfx for avatars -const imgSfx = ".image" - -var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`) - -// Store defines interface to store and and load avatars -type Store interface { - Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name - Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader - ID(avatarID string) (id string) // unique id of stored avatar's data - Remove(avatarID string) error // remove avatar data - List() (ids []string, err error) // list all avatar ids - Close() error // close store -} - -// Migrate avatars between stores -func Migrate(dst Store, src Store) (int, error) { - ids, err := src.List() - if err != nil { - return 0, err - } - for _, id := range ids { - srcReader, _, err := src.Get(id) - if err != nil { - log.Printf("[WARN] can't get reader for avatar %s", id) - continue - } - if _, err = dst.Put(id, srcReader); err != nil { - log.Printf("[WARN] can't put avatar %s", id) - } - if err = srcReader.Close(); err != nil { - log.Printf("[WARN] failed to close avatar %s", id) - } - } - return len(ids), nil -} - -// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side -// (width or height) preserving aspect ratio. -// Returns original reader if resizing is not needed or failed. -func resize(reader io.Reader, limit int) io.Reader { - if reader == nil { - log.Print("[WARN] avatar resize(): reader is nil") - return nil - } - if limit <= 0 { - log.Print("[DEBUG] avatar resize(): limit should be greater than 0") - return reader - } - - var teeBuf bytes.Buffer - tee := io.TeeReader(reader, &teeBuf) - src, _, err := image.Decode(tee) - if err != nil { - log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err) - return &teeBuf - } - - bounds := src.Bounds() - w, h := bounds.Dx(), bounds.Dy() - if w <= limit && h <= limit || w <= 0 || h <= 0 { - log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size") - return &teeBuf - } - newW, newH := w*limit/h, limit - if w > h { - newW, newH = limit, h*limit/w - } - m := image.NewRGBA(image.Rect(0, 0, newW, newH)) - // Slower than `draw.ApproxBiLinear.Scale()` but better quality. - draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil) - - var out bytes.Buffer - if err = png.Encode(&out, m); err != nil { - log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err) - return &teeBuf - } - return &out -} - -// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed -func encodeID(val string) string { - if reValidAvatarID.MatchString(val) { - return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image - } - return store.EncodeID(val) -} diff --git a/backend/app/store/avatar/store_test.go b/backend/app/store/avatar/store_test.go deleted file mode 100644 index 377af11a..00000000 --- a/backend/app/store/avatar/store_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package avatar - -import ( - "bytes" - "image" - "io" - "io/ioutil" - "os" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestAvatarStore_resize(t *testing.T) { - checkC := func(t *testing.T, r io.Reader, cExp []byte) { - content, err := ioutil.ReadAll(r) - require.NoError(t, err) - assert.Equal(t, cExp, content) - } - - // Reader is nil. - resizedR := resize(nil, 100) - // assert.EqualError(t, err, "limit should be greater than 0") - assert.Nil(t, resizedR) - - // Negative limit error. - resizedR = resize(strings.NewReader("some picture bin data"), -1) - require.NotNil(t, resizedR) - checkC(t, resizedR, []byte("some picture bin data")) - - // Decode error. - resizedR = resize(strings.NewReader("invalid image content"), 100) - assert.NotNil(t, resizedR) - checkC(t, resizedR, []byte("invalid image content")) - - cases := []struct { - file string - wr, hr int - }{ - {"testdata/circles.png", 400, 300}, // full size: 800x600 px - {"testdata/circles.jpg", 300, 400}, // full size: 600x800 px - } - - for _, c := range cases { - img, err := ioutil.ReadFile(c.file) - require.Nil(t, err, "can't open test file %s", c.file) - - // No need for resize, avatar dimensions are smaller than resize limit. - resizedR = resize(bytes.NewReader(img), 800) - assert.NotNilf(t, resizedR, "file %s", c.file) - checkC(t, resizedR, img) - - // Resizing to half of width. Check resizedR avatar format PNG. - resizedR = resize(bytes.NewReader(img), 400) - assert.NotNilf(t, resizedR, "file %s", c.file) - - imgRz, format, err := image.Decode(resizedR) - assert.Nilf(t, err, "file %s", c.file) - assert.Equalf(t, "png", format, "file %s", c.file) - bounds := imgRz.Bounds() - assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file) - assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file) - } -} - -func TestAvatarStore_Migrate(t *testing.T) { - // prep localfs - plocal := NewLocalFS("/tmp/avatars.test", 300) - err := os.MkdirAll("/tmp/avatars.test", 0700) - require.NoError(t, err) - defer os.RemoveAll("/tmp/avatars.test") - - // prep gridfs - pgfs, skip := prepGFStore(t) - if skip { - return - } - - // write to localfs - _, err = plocal.Put("user1", strings.NewReader("some picture bin data 1")) - require.Nil(t, err) - _, err = plocal.Put("user2", strings.NewReader("some picture bin data 2")) - require.Nil(t, err) - _, err = plocal.Put("user3", strings.NewReader("some picture bin data 3")) - require.Nil(t, err) - - // migrate and check reported count - count, err := Migrate(pgfs, plocal) - require.NoError(t, err) - assert.Equal(t, 3, count, "all 3 recs migrated") - - // list avatars - l, err := pgfs.List() - assert.NoError(t, err) - assert.Equal(t, 3, len(l), "3 avatars listed in destination store") - sort.Strings(l) - assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) - - // try to read one of migrated avatars - r, size, err := pgfs.Get("0b7f849446d3383546d15a480966084442cd2193.image") - assert.Nil(t, err) - assert.Equal(t, 23, size) - data, err := ioutil.ReadAll(r) - assert.Nil(t, err) - assert.Equal(t, "some picture bin data 3", string(data)) -} diff --git a/backend/app/store/avatar/testdata/circles.jpg b/backend/app/store/avatar/testdata/circles.jpg deleted file mode 100644 index 2c7048c3..00000000 Binary files a/backend/app/store/avatar/testdata/circles.jpg and /dev/null differ diff --git a/backend/app/store/avatar/testdata/circles.png b/backend/app/store/avatar/testdata/circles.png deleted file mode 100644 index fb30946d..00000000 Binary files a/backend/app/store/avatar/testdata/circles.png and /dev/null differ