mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/avatar_bolt (#205)

Browse Source 
* add bolt implementation of avatar store

* include bolt avatar store in main's test

* change docs with new avatart params

* lint: single line declaration

* separate bucket for avatars metas

* fix comments

* test for bolt params

* closable avatar store

* fix dbl-close
This commit is contained in:
Umputun
2018-09-14 20:33:40 -05:00
committed by GitHub
parent d7185a861a
commit c70e060d22
10 changed files with 337 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+39 -40
View File
@@ -82,46 +82,45 @@ _this is the recommended way to run remark42_
#### Parameters
| Command line | Environment | Default | Description |
| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
| url | REMARK_URL | | url to remark42 server, _required_ |
| secret | SECRET | | secret key, _required_ |
| site | SITE | `remark` | site name(s), _multi_ |
| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| store.mongo.url | STORE_MONGO_URL | | mongo url for data store |
| store.mongo.db | STORE_MONGO_DB | | mongo db for data store |
| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ |
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs` or `mongo` |
| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store |
| avatar.mongo.url | AVATAR_MONGO_URL | | mongo url for avatar store |
| avatar.mongo.db | AVATAR_MONGO_DB | | mongo db for avatar store |
| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit |
| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL |
| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL |
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret |
| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only |
| low-score | LOW_SCORE | `-5` | low score threshold |
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
| edit-time | EDIT_TIME | `5m` | edit window |
| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images |
| dbg | DEBUG | `false` | debug mode |
| dev-passwd | DEV_PASSWD | | password for `dev` user |
| Command line | Environment | Default | Description |
| ------------------ | ------------------ | --------------------- | ------------------------------------------------ |
| url | REMARK_URL | | url to remark42 server, _required_ |
| secret | SECRET | | secret key, _required_ |
| site | SITE | `remark` | site name(s), _multi_ |
| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| mongo.url | MONGO_URL | | mongo url for all stores using mongodb |
| mongo.db | MONGO_DB | | mongo database |
| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ |
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, 'bolt`, or `mongo` |
| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store |
| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store |
| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL |
| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL |
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret |
| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only |
| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit |
| low-score | LOW_SCORE | `-5` | low score threshold |
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
| edit-time | EDIT_TIME | `5m` | edit window |
| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images |
| dbg | DEBUG | `false` | debug mode |
| dev-passwd | DEV_PASSWD | | password for `dev` user |
* command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
* _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
backend/app/cmd/avatar.go
+16
View File
@@ -2,8 +2,10 @@ package cmd
import (
"log"
"path"
"time"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
@@ -54,11 +56,20 @@ func (ac *AvatarCommand) Execute(args []string) error {
if err != nil {
return err
}
if err = dst.Close(); err != nil {
log.Printf("[WARN] failed to close dst store %s", ac.AvatarDst.Type)
}
if err = src.Close(); err != nil {
log.Printf("[WARN] failed to close src store %s", ac.AvatarSrc.Type)
}
log.Printf("[INFO] completed, migrated avatars = %d", count)
return nil
}
func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
log.Printf("[DEBUG] make avatar store, type=%s", gr.Type)
switch gr.Type {
case "fs":
if err := makeDirs(gr.FS.Path); err != nil {
@@ -72,6 +83,11 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
}
conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
return avatar.NewGridFS(conn, gr.RszLmt), nil
case "bolt":
if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
return nil, err
}
return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{}, gr.RszLmt)
}
return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
}
backend/app/cmd/avatar_test.go
+12
View File
@@ -22,6 +22,7 @@ func TestAvatar_Execute(t *testing.T) {
}
defer os.RemoveAll("/tmp/ava-test")
// from fs to mongo
cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
@@ -31,6 +32,17 @@ func TestAvatar_Execute(t *testing.T) {
err = cmd.Execute(nil)
assert.NoError(t, err)
// from fs to bolt
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
"--dst.bolt.file=/tmp/ava-test.db"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
// failed
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
backend/app/cmd/server.go
+16 -2
View File
@@ -7,6 +7,7 @@ import (
"net/url"
"os"
"os/signal"
"path"
"strings"
"syscall"
"time"
@@ -80,10 +81,13 @@ type StoreGroup struct {
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
Bolt struct {
File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
@@ -120,6 +124,7 @@ type serverApp struct {
exporter migrator.Exporter
devAuth *auth.DevAuthServer
dataService *service.DataStore
avatarStore avatar.Store
terminated chan struct{}
}
@@ -247,6 +252,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
exporter: exporter,
devAuth: devAuth,
dataService: dataService,
avatarStore: avatarStore,
terminated: make(chan struct{}),
}, nil
}
@@ -265,7 +271,10 @@ func (a *serverApp) run(ctx context.Context) error {
a.devAuth.Shutdown()
}
if e := a.dataService.Close(); e != nil {
log.Printf("[WARN] failed to close store, %s", e)
log.Printf("[WARN] failed to close data store, %s", e)
}
if e := a.avatarStore.Close(); e != nil {
log.Printf("[WARN] failed to close avatar store, %s", e)
}
}()
@@ -340,6 +349,11 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
return avatar.NewGridFS(conn, s.Avatar.RszLmt), nil
case "bolt":
if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
return nil, err
}
return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{}, s.Avatar.RszLmt)
}
return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
}
backend/app/cmd/server_test.go
+3 -1
View File
@@ -202,7 +202,9 @@ func TestServerApp_MainSignal(t *testing.T) {
s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
p := flags.NewParser(&s, flags.Default)
args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp", "--port=18100"}
args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100"}
defer os.Remove("/tmp/ava-test.db")
_, err := p.ParseArgs(args)
require.Nil(t, err)
err = s.Execute(args)
backend/app/store/avatar/bolt.go
+142
View File
@@ -0,0 +1,142 @@
package avatar
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"io"
"io/ioutil"
"log"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// BoltDB implements avatar store with bolt
// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket to keep sha1
// avatarID (base file name) used as a key
type BoltDB struct {
fileName string // full path to boltdb
resizeLimit int
db *bolt.DB
}
const avatarsBktName = "avatars"
const metasBktName = "metas"
// NewBoltDB makes bolt avatar store
func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, error) {
db, err := bolt.Open(fileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
}
err = db.Update(func(tx *bolt.Tx) error {
if _, e := tx.CreateBucketIfNotExists([]byte(avatarsBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", avatarsBktName)
}
_, e := tx.CreateBucketIfNotExists([]byte(metasBktName))
return errors.Wrapf(e, "failed to create top metas bucket %s", metasBktName)
})
if err != nil {
return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
}
return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil
}
// Put avatar to bolt, key by avatarID
func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
// Trying to resize avatar.
if reader = resize(reader, b.resizeLimit); reader == nil {
return "", errors.New("avatar resize reader is nil")
}
avatarID := id + imgSfx
err = b.db.Update(func(tx *bolt.Tx) error {
buf := &bytes.Buffer{}
if _, err = io.Copy(buf, reader); err != nil {
return errors.Wrapf(err, "can't read avatar %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", avatarID)
}
return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), id)))
})
return avatarID, err
}
// Get avatar reader for avatar id.image, avatarID used as the direct key
func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error) {
buf := &bytes.Buffer{}
err = b.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(avatarsBktName)).Get([]byte(avatarID))
if data == nil {
return errors.Errorf("can't load avatar %s", avatarID)
}
size, err = buf.Write(data)
return errors.Wrapf(err, "failed to write for %s", avatarID)
})
return ioutil.NopCloser(buf), size, err
}
// ID returns a fingerprint of the avatar content.
func (b *BoltDB) ID(avatarID string) (id string) {
data := []byte{}
err := b.db.View(func(tx *bolt.Tx) error {
data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID))
if data == nil {
return errors.Errorf("can't load avatar's id for %s", avatarID)
}
return nil
})
if err != nil {
log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return string(data)
}
// Remove avatar from bolt
func (b *BoltDB) Remove(avatarID string) (err error) {
return b.db.Update(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte(avatarsBktName))
if bkt.Get([]byte(avatarID)) == nil {
return errors.Errorf("avatar key not found, %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Delete([]byte(avatarID)); err != nil {
return errors.Wrapf(err, "can't delete avatar object %s", avatarID)
}
return errors.Wrapf(tx.Bucket([]byte(metasBktName)).Delete([]byte(avatarID)),
"can't delete meta object %s", avatarID)
})
}
// List all avatars (ids) from metas bucket
// note: id includes .image suffix
func (b *BoltDB) List() (ids []string, err error) {
err = b.db.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(metasBktName)).ForEach(func(k, _ []byte) error {
ids = append(ids, string(k))
return nil
})
})
return ids, errors.Wrap(err, "failed to list")
}
// Close bolt store
func (b *BoltDB) Close() error {
return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName)
}
func (b *BoltDB) sha1(data []byte, avatarID string) (id string) {
h := sha1.New()
if _, err := h.Write(data); err != nil {
log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return hex.EncodeToString(h.Sum(nil))
}
backend/app/store/avatar/bolt_test.go
+98
View File
@@ -0,0 +1,98 @@
package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testDb = "/tmp/test-remark-avatars.db"
func TestBoltDB_PutAndGet(t *testing.T) {
var b Store = prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
rd, size, err := b.Get(avatar)
require.Nil(t, err)
assert.Equal(t, 21, size)
data, err := ioutil.ReadAll(rd)
require.Nil(t, err)
assert.Equal(t, "some picture bin data", string(data))
_, _, err = b.Get("bad avatar")
assert.NotNil(t, err)
assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encode avatar id")
l, err := b.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestBoltDB_Remove(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
assert.NotNil(t, b.Remove("no-such-thing.image"))
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
assert.NoError(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one")
assert.NotNil(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestBoltDB_List(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
// write some avatars
_, err := b.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = b.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = b.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := b.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := b.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
// makes new boltdb, put two records
func prepBoltStore(t *testing.T) *BoltDB {
os.Remove(testDb)
boltStore, err := NewBoltDB(testDb, bolt.Options{}, 0)
require.Nil(t, err)
return boltStore
}
backend/app/store/avatar/gridfs.go
+5
View File
@@ -118,3 +118,8 @@ func (gf *GridFS) List() (ids []string, err error) {
}
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (gf *GridFS) Close() error {
return nil
}
backend/app/store/avatar/localfs.go
+5
View File
@@ -113,6 +113,11 @@ func (fs *LocalFS) List() (ids []string, err error) {
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (fs *LocalFS) Close() error {
return nil
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
backend/app/store/avatar/store.go
+1 -1
View File
@@ -32,7 +32,7 @@ type Store interface {
ID(avatarID string) (id string) // unique id of stored avatar's data
Remove(avatarID string) error // remove avatar data
List() (ids []string, err error) // list all avatar ids
Close() error
}
// Migrate avatars between stores