add image storage

2019-03-10 19:40:14 -05:00
parent 034101fb64
commit a1450cff57
2 changed files with 202 additions and 0 deletions
@@ -0,0 +1,108 @@
+// Package image handles storing, resizing and retrival of images
+package image
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"math"
+	"os"
+	"path"
+	"strconv"
+	"sync"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/pkg/errors"
+)
+
+// Interface defines Save and Load methods
+type Interface interface {
+	Save(name string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Load(id string) (io.ReadCloser, error)                // load image by ID. Caller has to close the reader.
+}
+
+// FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
+type FileSystem struct {
+	Location  string
+	MaxSize   int
+	Partitons int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS. Returns id as a hash of name
+// name should be passed in unique prefix, for example with userID_*
+func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
+
+	h := sha1.Sum([]byte(name))
+	id = hex.EncodeToString(h[:])
+	if ext := path.Ext(name); ext != "" {
+		id += ext
+	}
+	location := f.location(id)
+	dst := path.Join(location, id)
+
+	if err := os.MkdirAll(location, 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	fh, err := os.Create(dst)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't make image file %s", dst)
+	}
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	written, err := io.Copy(fh, lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+	if err := fh.Close(); err != nil {
+		return "", errors.Wrapf(err, "can't close image file %s", dst)
+	}
+	if written > int64(f.MaxSize) {
+		if err = os.Remove(dst); err != nil {
+			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
+		}
+		return "", errors.Errorf("file %s is too large", name)
+	}
+	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), name)
+	return id, nil
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, error) {
+	location := f.location(id)
+	imgFile := path.Join(location, id)
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, nil
+}
+
+// get location (directory) for id by adding partition to the final path in order to keep files
+// in different subdirectories and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/92. Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(id string) string {
+	if f.Partitons == 0 {
+		return f.Location
+	}
+
+	f.crc.Do(func() {
+		f.crc.Table = crc64.MakeTable(crc64.ECMA)
+		p := int(math.Round(math.Log10(float64(f.Partitons))))
+		f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+		f.crc.divider = uint64(math.Pow(10, float64(p)))
+	})
+	checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+	partition := checksum64 % f.crc.divider
+	return path.Join(f.Location, fmt.Sprintf(f.crc.mask, partition))
+}
@@ -0,0 +1,94 @@
+package image
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImage_Save(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   50,
+	}
+	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	assert.Equal(t, "6851dcde6024e03258a66705f29e14b506048c74.png", id)
+
+	dst := path.Join(loc, "02", id)
+	data, err := ioutil.ReadFile(dst)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestImage_SaveTooLarge(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   5,
+	}
+	_, err = svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.Error(t, err)
+	assert.EqualError(t, err, "file blah_ff1.png is too large")
+}
+
+func TestImage_Load(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	// save image
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   50,
+	}
+	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+
+	r, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer r.Close()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+
+	_, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestImage_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "abcdefg", "/tmp/2"},
+		{10, "abcdefe", "/tmp/1"},
+		{10, "12345", "/tmp/9"},
+		{100, "12345", "/tmp/69"},
+		{100, "xyzz", "/tmp/58"},
+		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/02"},
+		{0, "12345", "/tmp"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitons: tt.partitions}
+			assert.Equal(t, tt.res, svc.location(tt.id))
+		})
+	}
+}