diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go index 38a10617..eb067b1c 100644 --- a/backend/app/store/avatar/gridfs.go +++ b/backend/app/store/avatar/gridfs.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "log" + "time" "github.com/globalsign/mgo" "github.com/go-pkgz/mongo" @@ -26,7 +27,7 @@ type GridFS struct { // Put avatar to gridfs object, try to resize func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := store.EncodeID(userID) + id := encodeID(userID) err = gf.Connection.WithDB(func(dbase *mgo.Database) error { fh, e := dbase.GridFS("fs").Create(id + imgSfx) if e != nil { @@ -95,3 +96,25 @@ func (gf *GridFS) Remove(avatar string) error { return dbase.GridFS("fs").Remove(avatar) }) } + +func (gf *GridFS) List() (ids []string, err error) { + + type gfsFile struct { + Id interface{} `bson:"_id"` + ChunkSize int `bson:"chunkSize"` + UploadDate time.Time `bson:"uploadDate"` + Length int64 `bson:",minsize"` + MD5 string + Filename string `bson:",omitempty"` + } + + files := []gfsFile{} + err = gf.Connection.WithDB(func(dbase *mgo.Database) error { + return dbase.GridFS("fs").Find(nil).All(&files) + }) + + for _, f := range files { + ids = append(ids, f.Filename) + } + return ids, errors.Wrap(err, "can't list avatars") +} diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go index 0ba335f0..4ca28741 100644 --- a/backend/app/store/avatar/gridfs_test.go +++ b/backend/app/store/avatar/gridfs_test.go @@ -2,6 +2,7 @@ package avatar import ( "io/ioutil" + "sort" "strings" "testing" @@ -32,6 +33,11 @@ func TestGridFS_PutAndGet(t *testing.T) { assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar)) assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id") + + l, err := p.List() + require.Nil(t, err) + assert.Equal(t, 1, len(l)) + assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0]) } func TestGridFS_Remove(t *testing.T) { @@ -49,6 +55,33 @@ func TestGridFS_Remove(t *testing.T) { assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed") } +func TestGridFS_List(t *testing.T) { + p, skip := prepGFStore(t) + if skip { + return + } + // write some avatars + _, err := p.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + l, err := p.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} + func prepGFStore(t *testing.T) (Store, bool) { conn, err := mongo.MakeTestConnection(t) if err != nil { diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go index 306c0221..6bcae627 100644 --- a/backend/app/store/avatar/localfs.go +++ b/backend/app/store/avatar/localfs.go @@ -7,6 +7,7 @@ import ( "log" "os" "path" + "path/filepath" "strconv" "strings" "sync" @@ -30,8 +31,9 @@ func NewLocalFS(storePath string, resizeLimit int) *LocalFS { } // Put avatar for userID to file and return avatar's file name (base), like 12345678.image +// userID can be avatarID as well, in this case encoding just strip .image prefix func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) { - id := store.EncodeID(userID) + id := encodeID(userID) location := fs.location(id) // location adds partition to path if _, err = os.Stat(location); os.IsNotExist(err) { @@ -95,6 +97,20 @@ func (fs *LocalFS) Remove(avatar string) error { return os.Remove(avFile) } +func (fs *LocalFS) List() (ids []string, err error) { + err = filepath.Walk(fs.storePath, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) { + ids = append(ids, info.Name()) + } + return nil + }) + return ids, errors.Wrap(err, "can't list avatars") +} + // get location (directory) for user id by adding partition to final path in order to keep files // in different subdirectories and avoid too many files in a single place. // the end result is a full path like this - /tmp/avatars.test/92 diff --git a/backend/app/store/avatar/localfs_test.go b/backend/app/store/avatar/localfs_test.go index 2b9d1de7..922c420f 100644 --- a/backend/app/store/avatar/localfs_test.go +++ b/backend/app/store/avatar/localfs_test.go @@ -3,6 +3,7 @@ package avatar import ( "io/ioutil" "os" + "sort" "strings" "testing" "time" @@ -35,6 +36,14 @@ func TestAvatarStoreFS_Put(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(25), fi.Size()) + // with encoded id + avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123")) + require.Nil(t, err) + assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar) + fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image") + assert.NoError(t, err) + assert.Equal(t, int64(25), fi.Size()) + // with resize file, e := os.Open("testdata/circles.png") require.Nil(t, e) @@ -84,6 +93,7 @@ func TestAvatarStoreFS_Location(t *testing.T) { {"abc", "/tmp/avatars.test/35"}, {"xyz", "/tmp/avatars.test/69"}, {"blah blah", "/tmp/avatars.test/29"}, + {"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"}, } for i, tt := range tbl { @@ -126,6 +136,34 @@ func TestAvatarStoreFS_Remove(t *testing.T) { t.Log(err) } +func TestAvatarStoreFS_List(t *testing.T) { + p := NewLocalFS("/tmp/avatars.test", 300) + err := os.MkdirAll("/tmp/avatars.test", 0700) + require.NoError(t, err) + defer os.RemoveAll("/tmp/avatars.test") + + // write some avatars + _, err = p.Put("user1", strings.NewReader("some picture bin data 1")) + require.Nil(t, err) + _, err = p.Put("user2", strings.NewReader("some picture bin data 2")) + require.Nil(t, err) + _, err = p.Put("user3", strings.NewReader("some picture bin data 3")) + require.Nil(t, err) + + l, err := p.List() + assert.NoError(t, err) + assert.Equal(t, 3, len(l), "3 avatars listed") + sort.Strings(l) + assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l) + + r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image") + assert.Nil(t, err) + assert.Equal(t, 23, size) + data, err := ioutil.ReadAll(r) + assert.Nil(t, err) + assert.Equal(t, "some picture bin data 3", string(data)) +} + func BenchmarkAvatarStoreFS_ID(b *testing.B) { p := NewLocalFS("/tmp/avatars.test", 300) os.MkdirAll("/tmp/avatars.test/30", 0700) diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go index 94392405..0a5b5c94 100644 --- a/backend/app/store/avatar/store.go +++ b/backend/app/store/avatar/store.go @@ -6,26 +6,32 @@ package avatar import ( "bytes" "image" - "image/png" - "io" - "log" + "strings" // Initializing packages for supporting GIF and JPEG formats. _ "image/gif" _ "image/jpeg" + "image/png" + "io" + "log" + "regexp" + "github.com/umputun/remark/backend/app/store" "golang.org/x/image/draw" ) // imgSfx for avatars const imgSfx = ".image" +var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`) + // Store defines interface to store and and load avatars type Store interface { Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader ID(avatarID string) (id string) // unique id of stored avatar's data Remove(avatarID string) error // remove avatar data + List() (ids []string, err error) // list all avatar ids } @@ -71,3 +77,11 @@ func resize(reader io.Reader, limit int) io.Reader { } return &out } + +// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed +func encodeID(val string) string { + if reValidAvatarID.MatchString(val) { + return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image + } + return store.EncodeID(val) +}