diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 999d9792..b63b9d8b 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -482,6 +482,22 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { func (s *ServerCommand) makePicturesStore() (*image.Service, error) { switch s.Image.Type { + case "bolt": + boltImageStore, err := image.NewBoltStorage( + s.Image.Bolt.File, + s.Image.MaxSize, + s.Image.ResizeHeight, + s.Image.ResizeWidth, + bolt.Options{}, + ) + if err != nil { + return nil, err + } + return &image.Service{ + Store: boltImageStore, + ImageAPI: s.RemarkURL + "/api/v1/picture/", + TTL: 5 * s.EditDuration, // add extra time to image TTL for staging + }, nil case "fs": if err := makeDirs(s.Image.FS.Path); err != nil { return nil, err diff --git a/backend/app/store/image/bolt_store.go b/backend/app/store/image/bolt_store.go new file mode 100644 index 00000000..eb049b06 --- /dev/null +++ b/backend/app/store/image/bolt_store.go @@ -0,0 +1,165 @@ +package image + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "io/ioutil" + "path" + "time" + + bolt "github.com/coreos/bbolt" + log "github.com/go-pkgz/lgr" + "github.com/pkg/errors" +) + +const imagesStagedBktName = "imagesStaged" +const imagesBktName = "images" +const insertTimeBktName = "insertTimestamps" + +// Bolt provides image Store for images keeping data in bolt DB, restricts max size. +// It uses 3 buckets to manage images data. +// Two buckets contains image data (staged and committed images). Third bucket holds insertion timestamps. +type Bolt struct { + fileName string + db *bolt.DB + MaxSize int + MaxHeight int + MaxWidth int +} + +// Create Bolt Store. +func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, options bolt.Options) (*Bolt, error) { + db, err := bolt.Open(fileName, 0600, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName) + } + + err = db.Update(func(tx *bolt.Tx) error { + if _, e := tx.CreateBucketIfNotExists([]byte(imagesBktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", imagesBktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(imagesStagedBktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", imagesStagedBktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(insertTimeBktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", insertTimeBktName) + } + return nil + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName) + } + return &Bolt{ + db: db, + fileName: fileName, + MaxSize: maxSize, + MaxHeight: maxHeight, + MaxWidth: maxWidth, + }, nil +} + +// Save data from reader to staging bucket in DB +func (b *Bolt) Save(fileName string, userID string, r io.Reader) (id string, err error) { + data, err := readAndValidateImage(r, b.MaxSize) + if err != nil { + return "", errors.Wrapf(err, "can't load image %s", fileName) + } + + data, _ = resize(data, b.MaxWidth, b.MaxHeight) + + id = path.Join(userID, guid()) + + err = b.db.Update(func(tx *bolt.Tx) error { + if err = tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), data); err != nil { + return errors.Wrapf(err, "can't put to bucket with %s", id) + } + tsBuf := &bytes.Buffer{} + if err = binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil { + return errors.Wrapf(err, "can't serialize timestamp for %s", id) + } + if err = tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil { + return errors.Wrapf(err, "can't put to bucket with %s", id) + } + return err + }) + + return id, err +} + +// Commit file stored in staging bucket by copying it to permanent bucket +// Data from staging bucket not removed immediately, but would be removed on cleanup +func (b *Bolt) Commit(id string) error { + err := b.db.Update(func(tx *bolt.Tx) error { + data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id)) + if data == nil { + return errors.Errorf("failed to commit %s, not found in staging", id) + } + err := tx.Bucket([]byte(imagesBktName)).Put([]byte(id), data) + return errors.Wrapf(err, "can't put to bucket with %s", id) + }) + return err +} + +// Load image from DB +// returns ReadCloser and caller should call close after processing completed. +func (b *Bolt) Load(id string) (io.ReadCloser, int64, error) { + buf := &bytes.Buffer{} + var size int = 0 + err := b.db.View(func(tx *bolt.Tx) error { + data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id)) + if data == nil { + data = tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id)) + } + if data == nil { + return errors.Errorf("can't load image %s", id) + } + var err error + size, err = buf.Write(data) + return errors.Wrapf(err, "failed to write for %s", id) + }) + return ioutil.NopCloser(buf), int64(size), err +} + +// Cleanup runs scan of staging and removes old data based on ttl +func (b *Bolt) Cleanup(ctx context.Context, ttl time.Duration) error { + err := b.db.Update(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(insertTimeBktName)).Cursor() + + idsToRemove := [][]byte{} + + for id, tsData := c.First(); id != nil; id, tsData = c.Next() { + var ts int64 + err := binary.Read(bytes.NewReader(tsData), binary.LittleEndian, &ts) + if err != nil { + return errors.Wrapf(err, "failed to deserialize timestamp for %s", id) + } + + age := time.Since(time.Unix(0, ts)) + + if age > ttl { + log.Printf("[INFO] remove staging image %s, age %v", id, age) + idsToRemove = append(idsToRemove, id) + err := c.Delete() + if err != nil { + return errors.Wrapf(err, "failed to remove timestamp for %s", id) + } + } + } + imgBkt := tx.Bucket([]byte(imagesStagedBktName)) + for _, id := range idsToRemove { + err := imgBkt.Delete(id) + if err != nil { + return errors.Wrapf(err, "failed to remove image for %s", id) + } + } + return nil + }) + return err +} + +// SizeLimit returns max size of allowed image +func (b *Bolt) SizeLimit() int { + return b.MaxSize +} diff --git a/backend/app/store/image/bolt_store_test.go b/backend/app/store/image/bolt_store_test.go new file mode 100644 index 00000000..0b7a6d56 --- /dev/null +++ b/backend/app/store/image/bolt_store_test.go @@ -0,0 +1,152 @@ +package image + +import ( + "context" + "io/ioutil" + "os" + "path" + "testing" + "time" + + bolt "github.com/coreos/bbolt" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBoltStore_SaveCommit(t *testing.T) { + svc, teardown := prepareBoltImageStorageTest(t) + defer teardown() + + id, err := svc.Save("file1.png", "user1", gopherPNG()) + assert.NoError(t, err) + assert.Contains(t, id, "user1") + t.Log(id) + + err = svc.db.View(func(tx *bolt.Tx) error { + data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id)) + assert.NotNil(t, data) + assert.Equal(t, 1462, len(data)) + return nil + }) + assert.NoError(t, err) + + err = svc.Commit(id) + require.NoError(t, err) + + err = svc.db.View(func(tx *bolt.Tx) error { + data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id)) + assert.NotNil(t, data) + assert.Equal(t, 1462, len(data)) + return nil + }) + assert.NoError(t, err) + +} + +func TestBoltStore_LoadAfterSave(t *testing.T) { + svc, teardown := prepareBoltImageStorageTest(t) + defer teardown() + + id, err := svc.Save("file1.png", "user1", gopherPNG()) + assert.NoError(t, err) + assert.Contains(t, id, "user1") + t.Log(id) + + r, sz, err := svc.Load(id) + assert.NoError(t, err) + defer func() { assert.NoError(t, r.Close()) }() + data, err := ioutil.ReadAll(r) + + assert.NoError(t, err) + assert.Equal(t, 1462, len(data)) + assert.Equal(t, int64(1462), sz) + + _, _, err = svc.Load("abcd") + assert.NotNil(t, err) +} + +func TestBoltStore_Cleanup(t *testing.T) { + svc, teardown := prepareBoltImageStorageTest(t) + defer teardown() + + save := func(file string, user string) (id string) { + id, err := svc.Save(file, user, gopherPNG()) + require.NoError(t, err) + + checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error { + assert.NotNil(t, data) + assert.Equal(t, 1462, len(data)) + return nil + }) + return id + } + + // save 3 images to staging + img1 := save("blah_ff1.png", "user1") + time.Sleep(100 * time.Millisecond) + img2 := save("blah_ff2.png", "user1") + time.Sleep(100 * time.Millisecond) + img3 := save("blah_ff3.png", "user2") + + time.Sleep(100 * time.Millisecond) // make first image expired + err := svc.Cleanup(context.Background(), time.Millisecond*300) + assert.NoError(t, err) + + assertBoltImgNil(t, svc.db, imagesStagedBktName, img1) + assertBoltImgNil(t, svc.db, imagesBktName, img1) + assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img2) + assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img3) + + err = svc.Commit(img3) + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) // make all images except commited expired + err = svc.Cleanup(context.Background(), time.Millisecond*300) + assert.NoError(t, err) + + assertBoltImgNil(t, svc.db, imagesStagedBktName, img2) + assertBoltImgNil(t, svc.db, imagesBktName, img2) + assertBoltImgNotNil(t, svc.db, imagesBktName, img3) + assert.NoError(t, err) +} + +func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) { + checkBoltImgData(t, db, bucket, id, func(data []byte) error { + assert.Nil(t, data) + return nil + }) +} + +func assertBoltImgNotNil(t *testing.T, db *bolt.DB, bucket string, id string) { + checkBoltImgData(t, db, bucket, id, func(data []byte) error { + assert.NotNil(t, data) + return nil + }) +} + +func checkBoltImgData(t *testing.T, db *bolt.DB, bucket string, id string, callback func([]byte) error) { + err := db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(bucket)) + assert.NotNil(t, bkt, "bucket %s not found", bucket) + data := bkt.Get([]byte(id)) + return callback(data) + }) + assert.NoError(t, err) +} + +func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) { + loc, err := ioutil.TempDir("", "test_image_r42") + require.NoError(t, err, "failed to make temp dir") + + svc, err = NewBoltStorage(path.Join(loc, "picture.db"), 1500, 0, 0, bolt.Options{}) + assert.NoError(t, err, "new bolt storage") + + teardown = func() { + defer func() { + assert.NoError(t, os.RemoveAll(loc)) + }() + } + + return svc, teardown +} diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go index c1a9d12f..a1c21868 100644 --- a/backend/app/store/image/fs_store.go +++ b/backend/app/store/image/fs_store.go @@ -39,19 +39,9 @@ type FileSystem struct { // Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext // Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) { - - lr := io.LimitReader(r, int64(f.MaxSize)+1) - data, err := ioutil.ReadAll(lr) + data, err := readAndValidateImage(r, f.MaxSize) if err != nil { - return "", errors.Wrapf(err, "can't read source data for image %s", fileName) - } - if len(data) > f.MaxSize { - return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize) - } - - // read header first, needs it to check if data is valid png/gif/jpeg - if !isValidImage(data[:512]) { - return "", errors.Errorf("file %s is not in allowed format", fileName) + return "", errors.Wrapf(err, "can't load image %s", fileName) } data, resized := resize(data, f.MaxWidth, f.MaxHeight) diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go index a340af52..497d91a1 100644 --- a/backend/app/store/image/fs_store_test.go +++ b/backend/app/store/image/fs_store_test.go @@ -124,7 +124,7 @@ func TestFsStore_WrongFormat(t *testing.T) { defer teardown() _, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image")) - assert.EqualError(t, err, "file file1.png is not in allowed format") + assert.EqualError(t, err, "can't load image file1.png: file format is not allowed") } func TestFsStore_SaveAndCommit(t *testing.T) { diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index e9fd8d4f..0b9f32ce 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -13,6 +13,7 @@ import ( _ "image/jpeg" "image/png" "io" + "io/ioutil" "net/http" "strings" "sync" @@ -195,6 +196,25 @@ func isValidImage(b []byte) bool { return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp" } +func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) { + lr := io.LimitReader(r, int64(maxSize)+1) + data, err := ioutil.ReadAll(lr) + if err != nil { + return nil, err + } + + if len(data) > maxSize { + return nil, errors.Errorf("file is too large (limit=%d)", maxSize) + } + + // read header first, needs it to check if data is valid png/gif/jpeg + if !isValidImage(data[:512]) { + return nil, errors.Errorf("file format is not allowed") + } + + return data, nil +} + // guid makes a globally unique id func guid() string { return xid.New().String() diff --git a/frontend/app/components/input/_theme/_dark/input_theme_dark.scss b/frontend/app/components/input/_theme/_dark/input_theme_dark.scss index ba20e6ea..4a0c1e73 100644 --- a/frontend/app/components/input/_theme/_dark/input_theme_dark.scss +++ b/frontend/app/components/input/_theme/_dark/input_theme_dark.scss @@ -27,9 +27,9 @@ } .input__preview { - border-color: #eee; - background: #fff; - color: #333; + border-color: #22201c; + background: #22201c; + color: #ddd; } .input__preview-wrapper {