mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Umputun
2019-10-27 13:32:40 -05:00
parent 58f92062d9 cc5e8a6356
commit 6ad2263859
7 changed files with 359 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/cmd/server.go
+16
View File
@@ -482,6 +482,22 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
switch s.Image.Type {
case "bolt":
boltImageStore, err := image.NewBoltStorage(
s.Image.Bolt.File,
s.Image.MaxSize,
s.Image.ResizeHeight,
s.Image.ResizeWidth,
bolt.Options{},
)
if err != nil {
return nil, err
}
return &image.Service{
Store: boltImageStore,
ImageAPI: s.RemarkURL + "/api/v1/picture/",
TTL: 5 * s.EditDuration, // add extra time to image TTL for staging
}, nil
case "fs":
if err := makeDirs(s.Image.FS.Path); err != nil {
return nil, err
backend/app/store/image/bolt_store.go
+165
View File
@@ -0,0 +1,165 @@
package image
import (
"bytes"
"context"
"encoding/binary"
"io"
"io/ioutil"
"path"
"time"
bolt "github.com/coreos/bbolt"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
)
const imagesStagedBktName = "imagesStaged"
const imagesBktName = "images"
const insertTimeBktName = "insertTimestamps"
// Bolt provides image Store for images keeping data in bolt DB, restricts max size.
// It uses 3 buckets to manage images data.
// Two buckets contains image data (staged and committed images). Third bucket holds insertion timestamps.
type Bolt struct {
fileName string
db *bolt.DB
MaxSize int
MaxHeight int
MaxWidth int
}
// Create Bolt Store.
func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, options bolt.Options) (*Bolt, error) {
db, err := bolt.Open(fileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
}
err = db.Update(func(tx *bolt.Tx) error {
if _, e := tx.CreateBucketIfNotExists([]byte(imagesBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", imagesBktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(imagesStagedBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", imagesStagedBktName)
}
if _, e := tx.CreateBucketIfNotExists([]byte(insertTimeBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", insertTimeBktName)
}
return nil
})
if err != nil {
return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
}
return &Bolt{
db: db,
fileName: fileName,
MaxSize: maxSize,
MaxHeight: maxHeight,
MaxWidth: maxWidth,
}, nil
}
// Save data from reader to staging bucket in DB
func (b *Bolt) Save(fileName string, userID string, r io.Reader) (id string, err error) {
data, err := readAndValidateImage(r, b.MaxSize)
if err != nil {
return "", errors.Wrapf(err, "can't load image %s", fileName)
}
data, _ = resize(data, b.MaxWidth, b.MaxHeight)
id = path.Join(userID, guid())
err = b.db.Update(func(tx *bolt.Tx) error {
if err = tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), data); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", id)
}
tsBuf := &bytes.Buffer{}
if err = binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil {
return errors.Wrapf(err, "can't serialize timestamp for %s", id)
}
if err = tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", id)
}
return err
})
return id, err
}
// Commit file stored in staging bucket by copying it to permanent bucket
// Data from staging bucket not removed immediately, but would be removed on cleanup
func (b *Bolt) Commit(id string) error {
err := b.db.Update(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
if data == nil {
return errors.Errorf("failed to commit %s, not found in staging", id)
}
err := tx.Bucket([]byte(imagesBktName)).Put([]byte(id), data)
return errors.Wrapf(err, "can't put to bucket with %s", id)
})
return err
}
// Load image from DB
// returns ReadCloser and caller should call close after processing completed.
func (b *Bolt) Load(id string) (io.ReadCloser, int64, error) {
buf := &bytes.Buffer{}
var size int = 0
err := b.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
if data == nil {
data = tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
}
if data == nil {
return errors.Errorf("can't load image %s", id)
}
var err error
size, err = buf.Write(data)
return errors.Wrapf(err, "failed to write for %s", id)
})
return ioutil.NopCloser(buf), int64(size), err
}
// Cleanup runs scan of staging and removes old data based on ttl
func (b *Bolt) Cleanup(ctx context.Context, ttl time.Duration) error {
err := b.db.Update(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(insertTimeBktName)).Cursor()
idsToRemove := [][]byte{}
for id, tsData := c.First(); id != nil; id, tsData = c.Next() {
var ts int64
err := binary.Read(bytes.NewReader(tsData), binary.LittleEndian, &ts)
if err != nil {
return errors.Wrapf(err, "failed to deserialize timestamp for %s", id)
}
age := time.Since(time.Unix(0, ts))
if age > ttl {
log.Printf("[INFO] remove staging image %s, age %v", id, age)
idsToRemove = append(idsToRemove, id)
err := c.Delete()
if err != nil {
return errors.Wrapf(err, "failed to remove timestamp for %s", id)
}
}
}
imgBkt := tx.Bucket([]byte(imagesStagedBktName))
for _, id := range idsToRemove {
err := imgBkt.Delete(id)
if err != nil {
return errors.Wrapf(err, "failed to remove image for %s", id)
}
}
return nil
})
return err
}
// SizeLimit returns max size of allowed image
func (b *Bolt) SizeLimit() int {
return b.MaxSize
}
backend/app/store/image/bolt_store_test.go
+152
View File
@@ -0,0 +1,152 @@
package image
import (
"context"
"io/ioutil"
"os"
"path"
"testing"
"time"
bolt "github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBoltStore_SaveCommit(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
id, err := svc.Save("file1.png", "user1", gopherPNG())
assert.NoError(t, err)
assert.Contains(t, id, "user1")
t.Log(id)
err = svc.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
assert.NotNil(t, data)
assert.Equal(t, 1462, len(data))
return nil
})
assert.NoError(t, err)
err = svc.Commit(id)
require.NoError(t, err)
err = svc.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
assert.NotNil(t, data)
assert.Equal(t, 1462, len(data))
return nil
})
assert.NoError(t, err)
}
func TestBoltStore_LoadAfterSave(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
id, err := svc.Save("file1.png", "user1", gopherPNG())
assert.NoError(t, err)
assert.Contains(t, id, "user1")
t.Log(id)
r, sz, err := svc.Load(id)
assert.NoError(t, err)
defer func() { assert.NoError(t, r.Close()) }()
data, err := ioutil.ReadAll(r)
assert.NoError(t, err)
assert.Equal(t, 1462, len(data))
assert.Equal(t, int64(1462), sz)
_, _, err = svc.Load("abcd")
assert.NotNil(t, err)
}
func TestBoltStore_Cleanup(t *testing.T) {
svc, teardown := prepareBoltImageStorageTest(t)
defer teardown()
save := func(file string, user string) (id string) {
id, err := svc.Save(file, user, gopherPNG())
require.NoError(t, err)
checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error {
assert.NotNil(t, data)
assert.Equal(t, 1462, len(data))
return nil
})
return id
}
// save 3 images to staging
img1 := save("blah_ff1.png", "user1")
time.Sleep(100 * time.Millisecond)
img2 := save("blah_ff2.png", "user1")
time.Sleep(100 * time.Millisecond)
img3 := save("blah_ff3.png", "user2")
time.Sleep(100 * time.Millisecond) // make first image expired
err := svc.Cleanup(context.Background(), time.Millisecond*300)
assert.NoError(t, err)
assertBoltImgNil(t, svc.db, imagesStagedBktName, img1)
assertBoltImgNil(t, svc.db, imagesBktName, img1)
assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img2)
assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img3)
err = svc.Commit(img3)
require.NoError(t, err)
time.Sleep(200 * time.Millisecond) // make all images except commited expired
err = svc.Cleanup(context.Background(), time.Millisecond*300)
assert.NoError(t, err)
assertBoltImgNil(t, svc.db, imagesStagedBktName, img2)
assertBoltImgNil(t, svc.db, imagesBktName, img2)
assertBoltImgNotNil(t, svc.db, imagesBktName, img3)
assert.NoError(t, err)
}
func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) {
checkBoltImgData(t, db, bucket, id, func(data []byte) error {
assert.Nil(t, data)
return nil
})
}
func assertBoltImgNotNil(t *testing.T, db *bolt.DB, bucket string, id string) {
checkBoltImgData(t, db, bucket, id, func(data []byte) error {
assert.NotNil(t, data)
return nil
})
}
func checkBoltImgData(t *testing.T, db *bolt.DB, bucket string, id string, callback func([]byte) error) {
err := db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte(bucket))
assert.NotNil(t, bkt, "bucket %s not found", bucket)
data := bkt.Get([]byte(id))
return callback(data)
})
assert.NoError(t, err)
}
func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) {
loc, err := ioutil.TempDir("", "test_image_r42")
require.NoError(t, err, "failed to make temp dir")
svc, err = NewBoltStorage(path.Join(loc, "picture.db"), 1500, 0, 0, bolt.Options{})
assert.NoError(t, err, "new bolt storage")
teardown = func() {
defer func() {
assert.NoError(t, os.RemoveAll(loc))
}()
}
return svc, teardown
}
backend/app/store/image/fs_store.go
+2 -12
View File
@@ -39,19 +39,9 @@ type FileSystem struct {
// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
lr := io.LimitReader(r, int64(f.MaxSize)+1)
data, err := ioutil.ReadAll(lr)
data, err := readAndValidateImage(r, f.MaxSize)
if err != nil {
return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
}
if len(data) > f.MaxSize {
return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
}
// read header first, needs it to check if data is valid png/gif/jpeg
if !isValidImage(data[:512]) {
return "", errors.Errorf("file %s is not in allowed format", fileName)
return "", errors.Wrapf(err, "can't load image %s", fileName)
}
data, resized := resize(data, f.MaxWidth, f.MaxHeight)
backend/app/store/image/fs_store_test.go
+1 -1
View File
@@ -124,7 +124,7 @@ func TestFsStore_WrongFormat(t *testing.T) {
defer teardown()
_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
assert.EqualError(t, err, "file file1.png is not in allowed format")
assert.EqualError(t, err, "can't load image file1.png: file format is not allowed")
}
func TestFsStore_SaveAndCommit(t *testing.T) {
backend/app/store/image/image.go
+20
View File
@@ -13,6 +13,7 @@ import (
_ "image/jpeg"
"image/png"
"io"
"io/ioutil"
"net/http"
"strings"
"sync"
@@ -195,6 +196,25 @@ func isValidImage(b []byte) bool {
return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
}
func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) {
lr := io.LimitReader(r, int64(maxSize)+1)
data, err := ioutil.ReadAll(lr)
if err != nil {
return nil, err
}
if len(data) > maxSize {
return nil, errors.Errorf("file is too large (limit=%d)", maxSize)
}
// read header first, needs it to check if data is valid png/gif/jpeg
if !isValidImage(data[:512]) {
return nil, errors.Errorf("file format is not allowed")
}
return data, nil
}
// guid makes a globally unique id
func guid() string {
return xid.New().String()
frontend/app/components/input/_theme/_dark/input_theme_dark.scss
+3 -3
View File
@@ -27,9 +27,9 @@
}
.input__preview {
border-color: #eee;
background: #fff;
color: #333;
border-color: #22201c;
background: #22201c;
color: #ddd;
}
.input__preview-wrapper {