From a1450cff575b4ce24aa9450b84e164747984e0c9 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 10 Mar 2019 19:40:14 -0500 Subject: [PATCH] add image storage --- backend/app/store/image/image.go | 108 ++++++++++++++++++++++++++ backend/app/store/image/image_test.go | 94 ++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 backend/app/store/image/image.go create mode 100644 backend/app/store/image/image_test.go diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go new file mode 100644 index 00000000..80ec9fb2 --- /dev/null +++ b/backend/app/store/image/image.go @@ -0,0 +1,108 @@ +// Package image handles storing, resizing and retrival of images +package image + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "hash/crc64" + "io" + "math" + "os" + "path" + "strconv" + "sync" + + log "github.com/go-pkgz/lgr" + + "github.com/pkg/errors" +) + +// Interface defines Save and Load methods +type Interface interface { + Save(name string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image + Load(id string) (io.ReadCloser, error) // load image by ID. Caller has to close the reader. +} + +// FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size +type FileSystem struct { + Location string + MaxSize int + Partitons int + + crc struct { + *crc64.Table + sync.Once + mask string + divider uint64 + } +} + +// Save data from reader for given file name to local FS. Returns id as a hash of name +// name should be passed in unique prefix, for example with userID_* +func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) { + + h := sha1.Sum([]byte(name)) + id = hex.EncodeToString(h[:]) + if ext := path.Ext(name); ext != "" { + id += ext + } + location := f.location(id) + dst := path.Join(location, id) + + if err := os.MkdirAll(location, 0700); err != nil { + return "", errors.Wrap(err, "can't make image directory") + } + + fh, err := os.Create(dst) + if err != nil { + return "", errors.Wrapf(err, "can't make image file %s", dst) + } + lr := io.LimitReader(r, int64(f.MaxSize)+1) + written, err := io.Copy(fh, lr) + if err != nil { + return "", errors.Wrapf(err, "can't write image file %s", dst) + } + if err := fh.Close(); err != nil { + return "", errors.Wrapf(err, "can't close image file %s", dst) + } + if written > int64(f.MaxSize) { + if err = os.Remove(dst); err != nil { + log.Printf("[WARN] can't remove image file %s, %v", dst, err) + } + return "", errors.Errorf("file %s is too large", name) + } + log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), name) + return id, nil +} + +// Load image from FS. Uses id to get partition subdirectory. +// returns ReadCloser and caller should call close after processing completed. +func (f *FileSystem) Load(id string) (io.ReadCloser, error) { + location := f.location(id) + imgFile := path.Join(location, id) + fh, err := os.Open(imgFile) + if err != nil { + return nil, errors.Wrapf(err, "can't load image %s", id) + } + return fh, nil +} + +// get location (directory) for id by adding partition to the final path in order to keep files +// in different subdirectories and avoid too many files in a single place. +// the end result is a full path like this - /tmp/images/92. Number of partitions defined by FileSystem.Partitions +func (f *FileSystem) location(id string) string { + if f.Partitons == 0 { + return f.Location + } + + f.crc.Do(func() { + f.crc.Table = crc64.MakeTable(crc64.ECMA) + p := int(math.Round(math.Log10(float64(f.Partitons)))) + f.crc.mask = "%0" + strconv.Itoa(p) + "d" + f.crc.divider = uint64(math.Pow(10, float64(p))) + }) + checksum64 := crc64.Checksum([]byte(id), f.crc.Table) + partition := checksum64 % f.crc.divider + return path.Join(f.Location, fmt.Sprintf(f.crc.mask, partition)) +} diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go new file mode 100644 index 00000000..94f6358c --- /dev/null +++ b/backend/app/store/image/image_test.go @@ -0,0 +1,94 @@ +package image + +import ( + "io/ioutil" + "os" + "path" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImage_Save(t *testing.T) { + loc, err := ioutil.TempDir("", "test_image_r42") + require.NoError(t, err, "failed to make temp dir") + defer os.RemoveAll(loc) + + svc := FileSystem{ + Location: loc, + Partitons: 100, + MaxSize: 50, + } + id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah")) + assert.NoError(t, err) + assert.Equal(t, "6851dcde6024e03258a66705f29e14b506048c74.png", id) + + dst := path.Join(loc, "02", id) + data, err := ioutil.ReadFile(dst) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) +} + +func TestImage_SaveTooLarge(t *testing.T) { + loc, err := ioutil.TempDir("", "test_image_r42") + require.NoError(t, err, "failed to make temp dir") + defer os.RemoveAll(loc) + + svc := FileSystem{ + Location: loc, + Partitons: 100, + MaxSize: 5, + } + _, err = svc.Save("blah_ff1.png", strings.NewReader("blah blah")) + assert.Error(t, err) + assert.EqualError(t, err, "file blah_ff1.png is too large") +} + +func TestImage_Load(t *testing.T) { + loc, err := ioutil.TempDir("", "test_image_r42") + require.NoError(t, err, "failed to make temp dir") + defer os.RemoveAll(loc) + + // save image + svc := FileSystem{ + Location: loc, + Partitons: 100, + MaxSize: 50, + } + id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah")) + assert.NoError(t, err) + + r, err := svc.Load(id) + assert.NoError(t, err) + defer r.Close() + data, err := ioutil.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) + + _, err = svc.Load("abcd") + assert.NotNil(t, err) +} + +func TestImage_location(t *testing.T) { + tbl := []struct { + partitions int + id, res string + }{ + {10, "abcdefg", "/tmp/2"}, + {10, "abcdefe", "/tmp/1"}, + {10, "12345", "/tmp/9"}, + {100, "12345", "/tmp/69"}, + {100, "xyzz", "/tmp/58"}, + {100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/02"}, + {0, "12345", "/tmp"}, + } + for n, tt := range tbl { + t.Run(strconv.Itoa(n), func(t *testing.T) { + svc := FileSystem{Location: "/tmp", Partitons: tt.partitions} + assert.Equal(t, tt.res, svc.location(tt.id)) + }) + } +}