mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rest/api passes

Browse Source
This commit is contained in:
Umputun
2018-12-27 20:16:06 -06:00
parent b537f01b41
commit 59cdfc3d04
21 changed files with 83 additions and 1346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/rest/api/admin.go
+3 -4
View File
@@ -14,7 +14,6 @@ import (
"github.com/go-pkgz/rest/cache"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -25,7 +24,6 @@ type admin struct {
cache cache.LoadingCache
authenticator auth.Service
readOnlyAge int
avatarProxy *proxy.Avatar
migrator *Migrator
}
@@ -111,7 +109,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.Audience)
// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
if val, err := claims.User.BoolAttr("delete_me"); err != nil || !val {
if !claims.User.BoolAttr("delete_me") {
rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
return
}
@@ -122,7 +120,8 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
}
if claims.User.Picture != "" {
if err := a.avatarProxy.Store.Remove(path.Base(claims.User.Picture)); err != nil {
avatartStore := a.authenticator.AvatarProxy().Store
if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
return
}
backend/app/rest/api/admin_test.go
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"testing"
"time"
"github.com/dgrijalva/jwt-go"
jwt "github.com/dgrijalva/jwt-go"
"github.com/go-pkgz/auth/token"
R "github.com/go-pkgz/rest"
@@ -615,7 +615,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
// try without deleteme flag
badClaims2 := claims
badClaims2.User.SetBoolAttr("delete_me", true)
badClaims2.User.SetBoolAttr("delete_me", false)
tkn, err = srv.Authenticator.TokenService().Token(badClaims2)
assert.Nil(t, err)
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
backend/app/rest/api/rest.go
+9 -4
View File
@@ -41,7 +41,6 @@ type Rest struct {
DataService *service.DataStore
Authenticator auth.Service
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
CommentFormatter *store.CommentFormatter
Migrator *Migrator
@@ -167,7 +166,6 @@ func (s *Rest) routes() chi.Router {
cache: s.Cache,
authenticator: s.Authenticator,
readOnlyAge: s.ReadOnlyAge,
avatarProxy: s.AvatarProxy,
}
corsMiddleware := cors.New(cors.Options{
@@ -219,9 +217,15 @@ func (s *Rest) routes() chi.Router {
// api routes
router.Route("/api/v1", func(rapi chi.Router) {
rapi.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
rapi.Group(func(rava chi.Router) {
rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
rava.Mount("/avatar", avatarHandler)
})
// open routes
rapi.Group(func(ropen chi.Router) {
ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
ropen.Use(authMiddleware.Trace)
ropen.Use(logger.New(logger.Flags(logger.All), logger.IPfn(ipFn)).Handler)
ropen.Get("/find", s.findCommentsCtrl)
@@ -241,6 +245,7 @@ func (s *Rest) routes() chi.Router {
// protected routes, require auth
rapi.Group(func(rauth chi.Router) {
rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
rauth.Use(authMiddleware.Auth)
rauth.Use(logger.New(logger.Flags(logger.All), logger.IPfn(ipFn)).Handler)
rauth.Post("/comment", s.createCommentCtrl)
@@ -255,7 +260,7 @@ func (s *Rest) routes() chi.Router {
})
})
// respond to /robots.tx with the list of allowed paths
// respond to /robots.txt with the list of allowed paths
router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
backend/app/rest/api/rest_private.go
+4 -3
View File
@@ -5,18 +5,19 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/go-pkgz/auth/token"
"log"
"net/http"
"strings"
"time"
"github.com/dgrijalva/jwt-go"
"github.com/go-pkgz/auth/token"
jwt "github.com/dgrijalva/jwt-go"
"github.com/go-chi/chi"
"github.com/go-chi/render"
R "github.com/go-pkgz/rest"
"github.com/go-pkgz/rest/cache"
"github.com/hashicorp/go-multierror"
multierror "github.com/hashicorp/go-multierror"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
backend/app/rest/api/rest_private_test.go
+5 -5
View File
@@ -142,8 +142,8 @@ func TestRest_CreateAndGet(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, "<p><strong>test</strong> <em>123</em></p>\n\n<p><a href=\"http://radio-t.com\" rel=\"nofollow\">http://radio-t.com</a></p>\n", comment.Text)
assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig)
assert.Equal(t, store.User{Name: "developer one", ID: "dev",
Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
assert.Equal(t, store.User{Name: "developer one", ID: "dev", Admin: true, Blocked: false,
IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
comment.User)
t.Logf("%+v", comment)
}
@@ -336,7 +336,7 @@ func TestRest_UserAllData(t *testing.T) {
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
assert.True(t, strings.HasPrefix(string(ungzBody),
`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
`{"info": {"name":"developer one","id":"dev","picture":"","admin":true}, "comments":[{`))
assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
t.Logf("%s", string(ungzBody))
@@ -347,7 +347,7 @@ func TestRest_UserAllData(t *testing.T) {
err = json.Unmarshal(ungzBody, &parsed)
assert.Nil(t, err)
assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true}, parsed.Info)
assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "", Admin: true}, parsed.Info)
assert.Equal(t, 3, len(parsed.Comments))
req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
@@ -387,7 +387,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
assert.True(t, strings.HasPrefix(string(ungzBody),
`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
`{"info": {"name":"developer one","id":"dev","picture":"","admin":true}, "comments":[{`))
assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
}
backend/app/rest/api/rest_public_test.go
+1 -2
View File
@@ -289,8 +289,7 @@ func TestRest_UserInfo(t *testing.T) {
user := store.User{}
err := json.Unmarshal([]byte(body), &user)
assert.Nil(t, err)
assert.Equal(t, store.User{Name: "developer one", ID: "dev",
Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: ""}, user)
assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "", Admin: true, Blocked: false, IP: ""}, user)
}
func TestRest_Count(t *testing.T) {
backend/app/rest/api/rest_test.go
+27 -30
View File
@@ -12,8 +12,10 @@ import (
"testing"
"time"
"github.com/coreos/bbolt"
bolt "github.com/coreos/bbolt"
"github.com/go-pkgz/auth"
"github.com/go-pkgz/auth/avatar"
"github.com/go-pkgz/auth/token"
R "github.com/go-pkgz/rest"
"github.com/go-pkgz/rest/cache"
"github.com/stretchr/testify/assert"
@@ -23,7 +25,6 @@ import (
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -61,8 +62,7 @@ func TestRest_GetStarted(t *testing.T) {
}
func TestRest_Shutdown(t *testing.T) {
srv := Rest{Authenticator: auth.Service{}, AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300),
RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
srv := Rest{Authenticator: auth.Service{}, ImageProxy: &proxy.Image{}}
go func() {
time.Sleep(100 * time.Millisecond)
@@ -91,11 +91,11 @@ func TestRest_filterComments(t *testing.T) {
func TestRest_RunStaticSSLMode(t *testing.T) {
srv := Rest{
Authenticator: auth.Service{},
AvatarProxy: &proxy.Avatar{
Store: avatar.NewLocalFS("/tmp", 300),
RoutePath: "/api/v1/avatar",
},
Authenticator: *auth.NewService(auth.Opts{
AvatarStore: avatar.NewLocalFS("/tmp"),
AvatarResizeLimit: 300,
}),
ImageProxy: &proxy.Image{},
SSLConfig: SSLConfig{
SSLMode: Static,
@@ -144,11 +144,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
srv := Rest{
Authenticator: auth.Service{},
AvatarProxy: &proxy.Avatar{
Store: avatar.NewLocalFS("/tmp", 300),
RoutePath: "/api/v1/avatar",
},
ImageProxy: &proxy.Image{},
ImageProxy: &proxy.Image{},
SSLConfig: SSLConfig{
SSLMode: Auto,
Port: 8443,
@@ -193,17 +189,18 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
MaxVotes: service.UnlimitedVotes,
}
//DevPasswd: "password",
// Providers: nil,
// KeyStore: adminStore,
// JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour),
srv = &Rest{
DataService: dataStore,
Authenticator: *auth.NewService(auth.Opts{}),
Cache: &cache.Nop{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
DataService: dataStore,
Authenticator: *auth.NewService(auth.Opts{
DevPasswd: "password",
SecretReader: token.SecretFunc(func(id string) (string, error) { return "secret", nil }),
AvatarStore: avatar.NewLocalFS("/tmp"),
AvatarResizeLimit: 300,
}),
Cache: &cache.Nop{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -257,21 +254,21 @@ func post(t *testing.T, url string, body string) (*http.Response, error) {
func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
b, err := json.Marshal(c)
assert.Nil(t, err, "can't marshal comment %+v", c)
require.Nil(t, err, "can't marshal comment %+v", c)
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
assert.Nil(t, err)
require.Nil(t, err)
req.SetBasicAuth("dev", "password")
resp, err := client.Do(req)
assert.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
require.Nil(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
b, err = ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
require.Nil(t, err)
crResp := R.JSON{}
err = json.Unmarshal(b, &crResp)
assert.Nil(t, err)
require.Nil(t, err)
time.Sleep(time.Nanosecond * 10)
return crResp["id"].(string)
}
backend/app/rest/proxy/avatar.go
-118
View File
@@ -1,118 +0,0 @@
package proxy
import (
"io"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
)
// Avatar provides http handler for avatars from avatar.Store
// On user login auth will call Put and it will retrieve and save picture locally.
type Avatar struct {
Store avatar.Store
RoutePath string
RemarkURL string
}
// Put stores retrieved avatar to avatar.Store. Gets image from user info. Returns proxied url
func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
// no picture for user, try default avatar
if u.Picture == "" {
return "", errors.Errorf("no picture for %s", u.ID)
}
// load avatar from remote location
client := http.Client{Timeout: 10 * time.Second}
var resp *http.Response
err = retry(5, time.Second, func() error {
var e error
resp, e = client.Get(u.Picture)
return e
})
if err != nil {
return "", errors.Wrap(err, "failed to fetch avatar from the orig")
}
defer func() {
if e := resp.Body.Close(); e != nil {
log.Printf("[WARN] can't close response body, %s", e)
}
}()
if resp.StatusCode != http.StatusOK {
return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status)
}
avatarID, err := p.Store.Put(u.ID, resp.Body) // put returns avatar base name, like 123456.image
if err != nil {
return "", err
}
log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatarID, u.Name)
return p.RemarkURL + p.RoutePath + "/" + avatarID, nil
}
// Routes returns auth routes for given provider
func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string, chi.Router) {
router := chi.NewRouter()
router.Use(middlewares...)
// GET /123456789.image
router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) {
avatarID := chi.URLParam(r, "avatar")
// enforce client-side caching
etag := `"` + p.Store.ID(avatarID) + `"`
w.Header().Set("Etag", etag)
w.Header().Set("Cache-Control", "max-age=604800") // 7 days
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, etag) {
w.WriteHeader(http.StatusNotModified)
return
}
}
avReader, size, err := p.Store.Get(avatarID)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar")
return
}
defer func() {
if e := avReader.Close(); e != nil {
log.Printf("[WARN] can't close avatar reader for %s, %s", avatarID, e)
}
}()
w.Header().Set("Content-Type", "image/*")
w.Header().Set("Content-Length", strconv.Itoa(size))
w.WriteHeader(http.StatusOK)
if _, err = io.Copy(w, avReader); err != nil {
log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
}
})
return p.RoutePath, router
}
func retry(retries int, delay time.Duration, fn func() error) (err error) {
for i := 0; i < retries; i++ {
if err = fn(); err == nil {
return nil
}
time.Sleep(delay)
}
return errors.Wrap(err, "retry failed")
}
backend/app/rest/proxy/avatar_test.go
-172
View File
@@ -1,172 +0,0 @@
package proxy
import (
"bytes"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
)
func TestAvatar_Put(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/pic.png" {
w.Header().Set("Content-Type", "image/*")
fmt.Fprint(w, "some picture bin data")
return
}
http.Error(w, "not found", http.StatusNotFound)
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
res, err := p.Put(u)
assert.NoError(t, err)
assert.Equal(t, "http://localhost:8080/avatar/b3daa77b4c04a9551b8781d03191fe098f325e67.image", res)
fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.NoError(t, err)
assert.Equal(t, int64(21), fi.Size())
u.ID = "user2"
res, err = p.Put(u)
assert.NoError(t, err)
assert.Equal(t, "http://localhost:8080/avatar/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", res)
fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
assert.NoError(t, err)
assert.Equal(t, int64(21), fi.Size())
}
func TestAvatar_PutFailed(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Print("request: ", r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
u := store.User{ID: "user1", Name: "user1 name"}
_, err := p.Put(u)
assert.EqualError(t, err, "no picture for user1")
u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"}
_, err = p.Put(u)
require.Error(t, err)
assert.Contains(t, err.Error(), "connect: connection refused")
u = store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/avatar/pic"}
_, err = p.Put(u)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to get avatar from the orig")
}
func TestAvatar_Routes(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/pic.png" {
w.Header().Set("Content-Type", "image/*")
w.Header().Set("Custom-Header", "xyz")
fmt.Fprint(w, "some picture bin data")
return
}
http.Error(w, "not found", http.StatusNotFound)
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
_, err := p.Put(u)
assert.NoError(t, err)
// status 400
req, err := http.NewRequest("GET", "/some_random_name.image", nil)
if err != nil {
t.Fatal(err)
}
rr := httptest.NewRecorder()
_, routes := p.Routes()
handler := http.Handler(routes)
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusBadRequest, rr.Code)
// status 200
req, err = http.NewRequest("GET", "/b3daa77b4c04a9551b8781d03191fe098f325e67.image", nil)
if err != nil {
t.Fatal(err)
}
rr = httptest.NewRecorder()
_, routes = p.Routes()
handler = http.Handler(routes)
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusOK, rr.Code)
assert.Equal(t, []string{"image/*"}, rr.HeaderMap["Content-Type"])
assert.Equal(t, []string{"21"}, rr.HeaderMap["Content-Length"])
assert.Equal(t, []string(nil), rr.HeaderMap["Custom-Header"], "strip all custom headers")
assert.NotNil(t, rr.HeaderMap["Etag"])
bb := bytes.Buffer{}
sz, err := io.Copy(&bb, rr.Body)
assert.NoError(t, err)
assert.Equal(t, int64(21), sz)
assert.Equal(t, "some picture bin data", bb.String())
// status 304
req, err = http.NewRequest("GET", "/some_random_name.image", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Add("If-None-Match", `"a008de0a2ccb3308b5d99ffff66436e15538f701"`) // hash of `some_random_name.image` since the file doesn't exist
rr = httptest.NewRecorder()
_, routes = p.Routes()
handler = http.Handler(routes)
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusNotModified, rr.Code)
assert.Equal(t, []string{`"a008de0a2ccb3308b5d99ffff66436e15538f701"`}, rr.HeaderMap["Etag"])
}
func TestAvatar_Retry(t *testing.T) {
i := 0
err := retry(5, time.Millisecond, func() error {
if i == 3 {
return nil
}
i++
return errors.New("err")
})
assert.Nil(t, err)
assert.Equal(t, 3, i)
st := time.Now()
err = retry(5, time.Millisecond, func() error {
return errors.New("err")
})
assert.NotNil(t, err)
assert.True(t, time.Since(st) >= time.Microsecond*5)
}
backend/app/rest/proxy/image.go
+3 -1
View File
@@ -8,6 +8,8 @@ import (
"strings"
"time"
"git.tkginternal.com/commons/pkg/repeater"
"github.com/PuerkitoBio/goquery"
"github.com/go-chi/chi"
"github.com/pkg/errors"
@@ -52,7 +54,7 @@ func (p Image) Routes() chi.Router {
client := http.Client{Timeout: 30 * time.Second}
var resp *http.Response
err = retry(5, time.Second, func() error {
err = repeater.NewDefault(5, time.Second).Do(func() error {
var e error
resp, e = client.Get(string(src))
return e
backend/app/rest/user.go
+29 -14
View File
@@ -1,15 +1,14 @@
package rest
import (
"context"
"errors"
"net/http"
"github.com/go-pkgz/auth/token"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
type contextKey string
// MustGetUserInfo fails if can't extract user data from the request.
// should be called from authed controllers only
func MustGetUserInfo(r *http.Request) store.User {
@@ -23,20 +22,36 @@ func MustGetUserInfo(r *http.Request) store.User {
// GetUserInfo returns user from request context
func GetUserInfo(r *http.Request) (user store.User, err error) {
ctx := r.Context()
if ctx == nil {
return store.User{}, errors.New("no info about user")
}
if u, ok := ctx.Value(contextKey("user")).(store.User); ok {
return u, nil
u, err := token.GetUserInfo(r)
if err != nil {
return store.User{}, errors.Wrap(err, "can't extract user info from the token")
}
return store.User{}, errors.New("user can't be parsed")
return store.User{
Name: u.Name,
ID: u.ID,
IP: u.IP,
Picture: u.Picture,
Admin: u.IsAdmin(),
Verified: u.BoolAttr("verified"),
Blocked: u.BoolAttr("blocked"),
}, nil
}
// SetUserInfo sets user into request context
func SetUserInfo(r *http.Request, user store.User) *http.Request {
ctx := r.Context()
ctx = context.WithValue(ctx, contextKey("user"), user)
return r.WithContext(ctx)
u := token.User{
ID: user.ID,
Name: user.Name,
Picture: user.Picture,
IP: user.IP,
Attributes: map[string]interface{}{
"blocked": user.Blocked,
"verified": user.Verified,
},
}
u.SetAdmin(user.Admin)
return token.SetUserInfo(r, u)
}
backend/app/store/avatar/bolt.go
-144
View File
@@ -1,144 +0,0 @@
package avatar
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"io"
"io/ioutil"
"log"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// BoltDB implements avatar store with bolt
// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket
// to keep sha1 of picture. avatarID (base file name) used as a key for both.
type BoltDB struct {
fileName string // full path to boltdb
resizeLimit int
db *bolt.DB
}
const avatarsBktName = "avatars"
const metasBktName = "metas"
// NewBoltDB makes bolt avatar store
func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, error) {
db, err := bolt.Open(fileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
}
err = db.Update(func(tx *bolt.Tx) error {
if _, e := tx.CreateBucketIfNotExists([]byte(avatarsBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", avatarsBktName)
}
_, e := tx.CreateBucketIfNotExists([]byte(metasBktName))
return errors.Wrapf(e, "failed to create top metas bucket %s", metasBktName)
})
if err != nil {
return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
}
return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil
}
// Put avatar to bolt, key by avatarID. Trying to resize image and lso calculates sha1 of the file for ID func
func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
// Trying to resize avatar.
if reader = resize(reader, b.resizeLimit); reader == nil {
return "", errors.New("avatar resize reader is nil")
}
avatarID := id + imgSfx
err = b.db.Update(func(tx *bolt.Tx) error {
buf := &bytes.Buffer{}
if _, err = io.Copy(buf, reader); err != nil {
return errors.Wrapf(err, "can't read avatar %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", avatarID)
}
// store sha1 of the image
return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID)))
})
return avatarID, err
}
// Get avatar reader for avatar id.image, avatarID used as the direct key
func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error) {
buf := &bytes.Buffer{}
err = b.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(avatarsBktName)).Get([]byte(avatarID))
if data == nil {
return errors.Errorf("can't load avatar %s", avatarID)
}
size, err = buf.Write(data)
return errors.Wrapf(err, "failed to write for %s", avatarID)
})
return ioutil.NopCloser(buf), size, err
}
// ID returns a fingerprint of the avatar content.
func (b *BoltDB) ID(avatarID string) (id string) {
data := []byte{}
err := b.db.View(func(tx *bolt.Tx) error {
if data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)); data == nil {
return errors.Errorf("can't load avatar's id for %s", avatarID)
}
return nil
})
if err != nil { // failed to get ID, use encoded avatarID
log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return string(data)
}
// Remove avatar from bolt
func (b *BoltDB) Remove(avatarID string) (err error) {
return b.db.Update(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte(avatarsBktName))
if bkt.Get([]byte(avatarID)) == nil {
return errors.Errorf("avatar key not found, %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Delete([]byte(avatarID)); err != nil {
return errors.Wrapf(err, "can't delete avatar object %s", avatarID)
}
return errors.Wrapf(tx.Bucket([]byte(metasBktName)).Delete([]byte(avatarID)),
"can't delete meta object %s", avatarID)
})
}
// List all avatars (ids) from metas bucket
// note: id includes .image suffix
func (b *BoltDB) List() (ids []string, err error) {
err = b.db.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(metasBktName)).ForEach(func(k, _ []byte) error {
ids = append(ids, string(k))
return nil
})
})
return ids, errors.Wrap(err, "failed to list")
}
// Close bolt store
func (b *BoltDB) Close() error {
return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName)
}
func (b *BoltDB) sha1(data []byte, avatarID string) (id string) {
h := sha1.New()
if _, err := h.Write(data); err != nil {
log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return hex.EncodeToString(h.Sum(nil))
}
backend/app/store/avatar/bolt_test.go
-99
View File
@@ -1,99 +0,0 @@
package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testDb = "/tmp/test-remark-avatars.db"
func TestBoltDB_PutAndGet(t *testing.T) {
var b Store = prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
rd, size, err := b.Get(avatar)
require.Nil(t, err)
assert.Equal(t, 21, size)
data, err := ioutil.ReadAll(rd)
require.Nil(t, err)
assert.Equal(t, "some picture bin data", string(data))
_, _, err = b.Get("bad avatar")
assert.NotNil(t, err)
// check IDs
assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encoded avatar id")
l, err := b.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestBoltDB_Remove(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
assert.NotNil(t, b.Remove("no-such-thing.image"))
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
assert.NoError(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one")
assert.NotNil(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestBoltDB_List(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
// write some avatars
_, err := b.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = b.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = b.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := b.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := b.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
// makes new boltdb, put two records
func prepBoltStore(t *testing.T) *BoltDB {
os.Remove(testDb)
boltStore, err := NewBoltDB(testDb, bolt.Options{}, 0)
require.Nil(t, err)
return boltStore
}
backend/app/store/avatar/gridfs.go
-125
View File
@@ -1,125 +0,0 @@
package avatar
import (
"bytes"
"io"
"io/ioutil"
"log"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// NewGridFS makes gridfs (mongo) avatar store
func NewGridFS(conn *mongo.Connection, resizeLimit int) *GridFS {
return &GridFS{Connection: conn, resizeLimit: resizeLimit}
}
// GridFS implements Store for GridFS
type GridFS struct {
Connection *mongo.Connection
resizeLimit int
}
// Put avatar to gridfs object, try to resize
func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Create(id + imgSfx)
if e != nil {
return e
}
defer func() {
if err = fh.Close(); err != nil {
log.Printf("[WARN] can't close avatar file %v, %s", fh, err)
}
}()
// Trying to resize avatar.
if reader = resize(reader, gf.resizeLimit); reader == nil {
return errors.New("avatar resize reader is nil")
}
_, e = io.Copy(fh, reader)
return e
})
return id + imgSfx, err
}
// Get avatar reader for avatar id.image
func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
buf := &bytes.Buffer{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't load avatar %s", avatar)
}
if _, e = io.Copy(buf, fh); e != nil {
return errors.Wrapf(e, "can't copy avatar %s", avatar)
}
size = int(fh.Size())
return fh.Close()
})
return ioutil.NopCloser(buf), size, err
}
// ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly
func (gf *GridFS) ID(avatar string) (id string) {
err := gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't open avatar %s", avatar)
}
id = fh.MD5()
return errors.Wrapf(fh.Close(), "can't close avatar")
})
if err != nil {
log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err)
return store.EncodeID(avatar)
}
return id
}
// Remove avatar from gridfs
func (gf *GridFS) Remove(avatar string) error {
return gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't get avatar %s", avatar)
}
if e = fh.Close(); e != nil {
log.Printf("[WARN] can't close avatar %s, %s", avatar, e)
}
return dbase.GridFS("fs").Remove(avatar)
})
}
// List all avatars (ids) on gfs
// note: id includes .image suffix
func (gf *GridFS) List() (ids []string, err error) {
type gfsFile struct {
UploadDate time.Time `bson:"uploadDate"`
Length int64 `bson:",minsize"`
MD5 string
Filename string `bson:",omitempty"`
}
files := []gfsFile{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
return dbase.GridFS("fs").Find(nil).All(&files)
})
for _, f := range files {
ids = append(ids, f.Filename)
}
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (gf *GridFS) Close() error {
return nil
}
backend/app/store/avatar/gridfs_test.go
-97
View File
@@ -1,97 +0,0 @@
package avatar
import (
"io/ioutil"
"sort"
"strings"
"testing"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGridFS_PutAndGet(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
avatar, err := p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
rd, size, err := p.Get(avatar)
require.Nil(t, err)
assert.Equal(t, 21, size)
data, err := ioutil.ReadAll(rd)
require.Nil(t, err)
assert.Equal(t, "some picture bin data", string(data))
_, _, err = p.Get("bad avatar")
assert.NotNil(t, err)
assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id")
l, err := p.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestGridFS_Remove(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
assert.NotNil(t, p.Remove("no-such-thing.image"))
avatar, err := p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one")
assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestGridFS_List(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
// write some avatars
_, err := p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func prepGFStore(t *testing.T) (Store, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
_ = conn.WithCustomCollection("fs.chunks", func(coll *mgo.Collection) error {
return coll.DropCollection()
})
_ = conn.WithCustomCollection("fs.files", func(coll *mgo.Collection) error {
return coll.DropCollection()
})
return NewGridFS(conn, 0), false
}
backend/app/store/avatar/localfs.go
-129
View File
@@ -1,129 +0,0 @@
package avatar
import (
"fmt"
"hash/crc64"
"io"
"log"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// LocalFS implements Store for local file system
type LocalFS struct {
storePath string
resizeLimit int
ctcTable *crc64.Table
once sync.Once
}
// NewLocalFS makes file-system avatar store
func NewLocalFS(storePath string, resizeLimit int) *LocalFS {
return &LocalFS{storePath: storePath, resizeLimit: resizeLimit}
}
// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
// userID can be avatarID as well, in this case encoding just strip .image prefix
func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
location := fs.location(id) // location adds partition to path
if _, err = os.Stat(location); os.IsNotExist(err) {
if e := os.Mkdir(location, 0700); e != nil {
return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
}
}
avFile := path.Join(location, id+imgSfx)
fh, err := os.Create(avFile)
if err != nil {
return "", errors.Wrapf(err, "can't create file %s", avFile)
}
defer func() {
if e := fh.Close(); e != nil {
log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
}
}()
// Trying to resize avatar.
if reader = resize(reader, fs.resizeLimit); reader == nil {
return "", errors.New("avatar resize reader is nil")
}
if _, err = io.Copy(fh, reader); err != nil {
return "", errors.Wrapf(err, "can't save file %s", avFile)
}
return id + imgSfx, nil
}
// Get avatar reader for avatar id.image
func (fs *LocalFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fh, err := os.Open(avFile)
if err != nil {
return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar)
}
if fi, e := fh.Stat(); e == nil {
size = int(fi.Size())
}
return fh, size, nil
}
// ID returns a fingerprint of the avatar content.
func (fs *LocalFS) ID(avatar string) (id string) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fi, err := os.Stat(avFile)
if err != nil {
log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err)
return store.EncodeID(avatar)
}
return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10))
}
// Remove avatar file
func (fs *LocalFS) Remove(avatar string) error {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
return os.Remove(avFile)
}
// List all avatars (ids) on local file system
// note: id includes .image suffix
func (fs *LocalFS) List() (ids []string, err error) {
err = filepath.Walk(fs.storePath,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) {
ids = append(ids, info.Name())
}
return nil
})
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (fs *LocalFS) Close() error {
return nil
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
func (fs *LocalFS) location(id string) string {
fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
partition := checksum64 % 100
return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
}
backend/app/store/avatar/localfs_test.go
-178
View File
@@ -1,178 +0,0 @@
package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAvatarStoreFS_Put(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
avatar, err := p.Put("user1", nil)
assert.Equal(t, "", avatar)
assert.EqualError(t, err, "avatar resize reader is nil")
avatar, err = p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.NoError(t, err)
assert.Equal(t, int64(21), fi.Size())
avatar, err = p.Put("user2", strings.NewReader("some picture bin data 123"))
require.Nil(t, err)
assert.Equal(t, "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with encoded id
avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123"))
require.Nil(t, err)
assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image")
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with resize
file, e := os.Open("testdata/circles.png")
require.Nil(t, e)
avatar, err = p.Put("user3", file)
require.Nil(t, err)
assert.Equal(t, "0b7f849446d3383546d15a480966084442cd2193.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/60/0b7f849446d3383546d15a480966084442cd2193.image")
assert.NoError(t, err)
assert.Equal(t, int64(6986), fi.Size())
p = NewLocalFS("/dev/null", 300)
_, err = p.Put("user1", strings.NewReader("some picture bin data"))
assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
}
func TestAvatarStoreFS_Get(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// file not exists
r, size, err := p.Get("some_random_name.image")
// nil, 0, errors.Wrapf(err, "can't load avatar %s, id")
assert.Nil(t, r)
assert.Equal(t, 0, size)
assert.EqualError(t, err, "can't load avatar some_random_name.image, id: open /tmp/avatars.test/91/some_random_name.image: no such file or directory")
// file exists
err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
assert.Nil(t, err)
r, size, err = p.Get("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.Nil(t, err)
assert.Equal(t, 9, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "something", string(data))
}
func TestAvatarStoreFS_Location(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
tbl := []struct {
id string
res string
}{
{"abc", "/tmp/avatars.test/35"},
{"xyz", "/tmp/avatars.test/69"},
{"blah blah", "/tmp/avatars.test/29"},
{"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"},
}
for i, tt := range tbl {
assert.Equal(t, tt.res, p.location(tt.id), "test #%d", i)
}
}
func TestAvatarStoreFS_ID(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// file not exists
id := p.ID("some_random_name.image")
assert.Equal(t, "a008de0a2ccb3308b5d99ffff66436e15538f701", id) // store.EncodeID("some_random_name.image")
// file exists
err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
require.NoError(t, err)
touch := time.Date(2017, 7, 14, 2, 40, 0, 0, time.UTC) // 1500000000
err = os.Chtimes("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", touch, touch)
require.NoError(t, err)
id = p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.Equal(t, "325d5b451f32c2f8e7f30a9fd65bff6a42954d9a", id) // store.EncodeID("b3daa77b4c04a9551b8781d03191fe098f325e67.image1500000000")
}
func TestAvatarStoreFS_Remove(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
assert.NotNil(t, p.Remove("no-such-avatar"), "remove non-existing avatar")
err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
require.NoError(t, err)
assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"))
_, err = os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.NotNil(t, err, "removed for real")
t.Log(err)
}
func TestAvatarStoreFS_List(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// write some avatars
_, err = p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func BenchmarkAvatarStoreFS_ID(b *testing.B) {
p := NewLocalFS("/tmp/avatars.test", 300)
os.MkdirAll("/tmp/avatars.test/30", 0700)
defer os.RemoveAll("/tmp/avatars.test")
err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
require.NoError(b, err)
b.ResetTimer()
for i := 0; i < b.N; i++ {
p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
}
}
backend/app/store/avatar/store.go
-110
View File
@@ -1,110 +0,0 @@
// Package avatar defines store interface and implements local (fs), gridfs (mongo) and boltdb stores.
//
package avatar
//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go"
import (
"bytes"
"image"
"strings"
// Initializing packages for supporting GIF and JPEG formats.
_ "image/gif"
_ "image/jpeg"
"image/png"
"io"
"log"
"regexp"
"github.com/umputun/remark/backend/app/store"
"golang.org/x/image/draw"
)
// imgSfx for avatars
const imgSfx = ".image"
var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`)
// Store defines interface to store and and load avatars
type Store interface {
Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name
Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader
ID(avatarID string) (id string) // unique id of stored avatar's data
Remove(avatarID string) error // remove avatar data
List() (ids []string, err error) // list all avatar ids
Close() error // close store
}
// Migrate avatars between stores
func Migrate(dst Store, src Store) (int, error) {
ids, err := src.List()
if err != nil {
return 0, err
}
for _, id := range ids {
srcReader, _, err := src.Get(id)
if err != nil {
log.Printf("[WARN] can't get reader for avatar %s", id)
continue
}
if _, err = dst.Put(id, srcReader); err != nil {
log.Printf("[WARN] can't put avatar %s", id)
}
if err = srcReader.Close(); err != nil {
log.Printf("[WARN] failed to close avatar %s", id)
}
}
return len(ids), nil
}
// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
// (width or height) preserving aspect ratio.
// Returns original reader if resizing is not needed or failed.
func resize(reader io.Reader, limit int) io.Reader {
if reader == nil {
log.Print("[WARN] avatar resize(): reader is nil")
return nil
}
if limit <= 0 {
log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
return reader
}
var teeBuf bytes.Buffer
tee := io.TeeReader(reader, &teeBuf)
src, _, err := image.Decode(tee)
if err != nil {
log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
return &teeBuf
}
bounds := src.Bounds()
w, h := bounds.Dx(), bounds.Dy()
if w <= limit && h <= limit || w <= 0 || h <= 0 {
log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
return &teeBuf
}
newW, newH := w*limit/h, limit
if w > h {
newW, newH = limit, h*limit/w
}
m := image.NewRGBA(image.Rect(0, 0, newW, newH))
// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
var out bytes.Buffer
if err = png.Encode(&out, m); err != nil {
log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
return &teeBuf
}
return &out
}
// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed
func encodeID(val string) string {
if reValidAvatarID.MatchString(val) {
return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image
}
return store.EncodeID(val)
}
backend/app/store/avatar/store_test.go
-109
View File
@@ -1,109 +0,0 @@
package avatar
import (
"bytes"
"image"
"io"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAvatarStore_resize(t *testing.T) {
checkC := func(t *testing.T, r io.Reader, cExp []byte) {
content, err := ioutil.ReadAll(r)
require.NoError(t, err)
assert.Equal(t, cExp, content)
}
// Reader is nil.
resizedR := resize(nil, 100)
// assert.EqualError(t, err, "limit should be greater than 0")
assert.Nil(t, resizedR)
// Negative limit error.
resizedR = resize(strings.NewReader("some picture bin data"), -1)
require.NotNil(t, resizedR)
checkC(t, resizedR, []byte("some picture bin data"))
// Decode error.
resizedR = resize(strings.NewReader("invalid image content"), 100)
assert.NotNil(t, resizedR)
checkC(t, resizedR, []byte("invalid image content"))
cases := []struct {
file string
wr, hr int
}{
{"testdata/circles.png", 400, 300}, // full size: 800x600 px
{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
}
for _, c := range cases {
img, err := ioutil.ReadFile(c.file)
require.Nil(t, err, "can't open test file %s", c.file)
// No need for resize, avatar dimensions are smaller than resize limit.
resizedR = resize(bytes.NewReader(img), 800)
assert.NotNilf(t, resizedR, "file %s", c.file)
checkC(t, resizedR, img)
// Resizing to half of width. Check resizedR avatar format PNG.
resizedR = resize(bytes.NewReader(img), 400)
assert.NotNilf(t, resizedR, "file %s", c.file)
imgRz, format, err := image.Decode(resizedR)
assert.Nilf(t, err, "file %s", c.file)
assert.Equalf(t, "png", format, "file %s", c.file)
bounds := imgRz.Bounds()
assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
}
}
func TestAvatarStore_Migrate(t *testing.T) {
// prep localfs
plocal := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// prep gridfs
pgfs, skip := prepGFStore(t)
if skip {
return
}
// write to localfs
_, err = plocal.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = plocal.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = plocal.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
// migrate and check reported count
count, err := Migrate(pgfs, plocal)
require.NoError(t, err)
assert.Equal(t, 3, count, "all 3 recs migrated")
// list avatars
l, err := pgfs.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed in destination store")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
// try to read one of migrated avatars
r, size, err := pgfs.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
backend/app/store/avatar/testdata/circles.jpg
Vendored
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 23 KiB

backend/app/store/avatar/testdata/circles.png
Vendored
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 11 KiB