mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add List to avatars and allow put without encoding

Browse Source
This commit is contained in:
Umputun
2018-09-10 12:19:52 -05:00
parent a856fd9c7d
commit 5b204e509e
5 changed files with 129 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/store/avatar/gridfs.go
+24 -1
View File
@@ -5,6 +5,7 @@ import (
"io"
"io/ioutil"
"log"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
@@ -26,7 +27,7 @@ type GridFS struct {
// Put avatar to gridfs object, try to resize
func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := store.EncodeID(userID)
id := encodeID(userID)
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Create(id + imgSfx)
if e != nil {
@@ -95,3 +96,25 @@ func (gf *GridFS) Remove(avatar string) error {
return dbase.GridFS("fs").Remove(avatar)
})
}
func (gf *GridFS) List() (ids []string, err error) {
type gfsFile struct {
Id interface{} `bson:"_id"`
ChunkSize int `bson:"chunkSize"`
UploadDate time.Time `bson:"uploadDate"`
Length int64 `bson:",minsize"`
MD5 string
Filename string `bson:",omitempty"`
}
files := []gfsFile{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
return dbase.GridFS("fs").Find(nil).All(&files)
})
for _, f := range files {
ids = append(ids, f.Filename)
}
return ids, errors.Wrap(err, "can't list avatars")
}
backend/app/store/avatar/gridfs_test.go
+33
View File
@@ -2,6 +2,7 @@ package avatar
import (
"io/ioutil"
"sort"
"strings"
"testing"
@@ -32,6 +33,11 @@ func TestGridFS_PutAndGet(t *testing.T) {
assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id")
l, err := p.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestGridFS_Remove(t *testing.T) {
@@ -49,6 +55,33 @@ func TestGridFS_Remove(t *testing.T) {
assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestGridFS_List(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
// write some avatars
_, err := p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func prepGFStore(t *testing.T) (Store, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
backend/app/store/avatar/localfs.go
+17 -1
View File
@@ -7,6 +7,7 @@ import (
"log"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
@@ -30,8 +31,9 @@ func NewLocalFS(storePath string, resizeLimit int) *LocalFS {
}
// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
// userID can be avatarID as well, in this case encoding just strip .image prefix
func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := store.EncodeID(userID)
id := encodeID(userID)
location := fs.location(id) // location adds partition to path
if _, err = os.Stat(location); os.IsNotExist(err) {
@@ -95,6 +97,20 @@ func (fs *LocalFS) Remove(avatar string) error {
return os.Remove(avFile)
}
func (fs *LocalFS) List() (ids []string, err error) {
err = filepath.Walk(fs.storePath,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) {
ids = append(ids, info.Name())
}
return nil
})
return ids, errors.Wrap(err, "can't list avatars")
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
backend/app/store/avatar/localfs_test.go
+38
View File
@@ -3,6 +3,7 @@ package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"time"
@@ -35,6 +36,14 @@ func TestAvatarStoreFS_Put(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with encoded id
avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123"))
require.Nil(t, err)
assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image")
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with resize
file, e := os.Open("testdata/circles.png")
require.Nil(t, e)
@@ -84,6 +93,7 @@ func TestAvatarStoreFS_Location(t *testing.T) {
{"abc", "/tmp/avatars.test/35"},
{"xyz", "/tmp/avatars.test/69"},
{"blah blah", "/tmp/avatars.test/29"},
{"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"},
}
for i, tt := range tbl {
@@ -126,6 +136,34 @@ func TestAvatarStoreFS_Remove(t *testing.T) {
t.Log(err)
}
func TestAvatarStoreFS_List(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// write some avatars
_, err = p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func BenchmarkAvatarStoreFS_ID(b *testing.B) {
p := NewLocalFS("/tmp/avatars.test", 300)
os.MkdirAll("/tmp/avatars.test/30", 0700)
backend/app/store/avatar/store.go
+17 -3
View File
@@ -6,26 +6,32 @@ package avatar
import (
"bytes"
"image"
"image/png"
"io"
"log"
"strings"
// Initializing packages for supporting GIF and JPEG formats.
_ "image/gif"
_ "image/jpeg"
"image/png"
"io"
"log"
"regexp"
"github.com/umputun/remark/backend/app/store"
"golang.org/x/image/draw"
)
// imgSfx for avatars
const imgSfx = ".image"
var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`)
// Store defines interface to store and and load avatars
type Store interface {
Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name
Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader
ID(avatarID string) (id string) // unique id of stored avatar's data
Remove(avatarID string) error // remove avatar data
List() (ids []string, err error) // list all avatar ids
}
@@ -71,3 +77,11 @@ func resize(reader io.Reader, limit int) io.Reader {
}
return &out
}
// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed
func encodeID(val string) string {
if reValidAvatarID.MatchString(val) {
return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image
}
return store.EncodeID(val)
}