diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go index fc35fd75..6af03440 100644 --- a/backend/app/store/image/fs_store.go +++ b/backend/app/store/image/fs_store.go @@ -1,6 +1,7 @@ package image import ( + "bytes" "context" "fmt" "hash/crc64" @@ -15,7 +16,6 @@ import ( "time" log "github.com/go-pkgz/lgr" - "github.com/google/uuid" "github.com/pkg/errors" ) @@ -38,12 +38,7 @@ type FileSystem struct { // Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) { - uid, err := uuid.NewUUID() - if err != nil { - return "", errors.Wrap(err, "can't make image uuid") - } - - id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext + id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext dst := f.location(f.Staging, id) if err = os.MkdirAll(path.Dir(dst), 0700); err != nil { @@ -54,20 +49,31 @@ func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id strin if err != nil { return "", errors.Wrapf(err, "can't make image file %s", dst) } + lr := io.LimitReader(r, int64(f.MaxSize)+1) - written, err := io.Copy(fh, lr) + + // read header first, needed it to check if data is valid png/gif/jpeg + header := make([]byte, 512) + hl, err := lr.Read(header) + if !isValidImage(header) { + return "", errors.Errorf("file %s is not in allowed format", fileName) + } + + written, err := io.Copy(fh, io.MultiReader(bytes.NewReader(header[:hl]), lr)) // write header and the rest of input if err != nil { return "", errors.Wrapf(err, "can't write image file %s", dst) } if err = fh.Close(); err != nil { return "", errors.Wrapf(err, "can't close image file %s", dst) } + if written > int64(f.MaxSize) { if err = os.Remove(dst); err != nil { log.Printf("[WARN] can't remove image file %s, %v", dst, err) } return "", errors.Errorf("file %s is too large", fileName) } + log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName) return id, nil } diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go index 39c2abad..333d853c 100644 --- a/backend/app/store/image/fs_store_test.go +++ b/backend/app/store/image/fs_store_test.go @@ -2,6 +2,8 @@ package image import ( "context" + "encoding/base64" + "io" "io/ioutil" "math/rand" "os" @@ -14,11 +16,16 @@ import ( "github.com/stretchr/testify/require" ) +// gopher png for test, from https://golang.org/src/image/png/example_test.go +const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==` + +func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } + func TestFsStore_Save(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah")) + id, err := svc.Save("file1.png", "user1", gopherPNG()) assert.NoError(t, err) assert.Contains(t, id, "user1/") assert.Contains(t, id, ".png") @@ -28,14 +35,22 @@ func TestFsStore_Save(t *testing.T) { t.Log(img) data, err := ioutil.ReadFile(img) assert.NoError(t, err) - assert.Equal(t, "blah blah", string(data)) + assert.Equal(t, 1462, len(data)) +} + +func TestFsStore_WrongFormat(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + + _, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image")) + assert.EqualError(t, err, "file file1.png is not in allowed format") } func TestFsStore_SaveAndCommit(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah")) + id, err := svc.Save("file1.png", "user1", gopherPNG()) require.NoError(t, err) err = svc.Commit(id) require.NoError(t, err) @@ -48,14 +63,14 @@ func TestFsStore_SaveAndCommit(t *testing.T) { t.Log(img) data, err := ioutil.ReadFile(img) assert.NoError(t, err) - assert.Equal(t, "blah blah", string(data)) + assert.Equal(t, 1462, len(data)) } func TestFsStore_SaveTooLarge(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - svc.MaxSize = 5 - _, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah")) + svc.MaxSize = 2000 + _, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG())) assert.Error(t, err) assert.Contains(t, err.Error(), "is too large") } @@ -65,7 +80,7 @@ func TestFsStore_LoadAfterSave(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah")) + id, err := svc.Save("blah_ff1.png", "user1", gopherPNG()) assert.NoError(t, err) t.Log(id) @@ -74,8 +89,8 @@ func TestFsStore_LoadAfterSave(t *testing.T) { defer func() { assert.NoError(t, r.Close()) }() data, err := ioutil.ReadAll(r) assert.NoError(t, err) - assert.Equal(t, "blah blah", string(data)) - assert.Equal(t, int64(9), sz) + assert.Equal(t, 1462, len(data)) + assert.Equal(t, int64(1462), sz) _, _, err = svc.Load("abcd") assert.NotNil(t, err) } @@ -85,7 +100,7 @@ func TestFsStore_LoadAfterCommit(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah")) + id, err := svc.Save("blah_ff1.png", "user1", gopherPNG()) assert.NoError(t, err) t.Log(id) err = svc.Commit(id) @@ -96,8 +111,8 @@ func TestFsStore_LoadAfterCommit(t *testing.T) { defer func() { assert.NoError(t, r.Close()) }() data, err := ioutil.ReadAll(r) assert.NoError(t, err) - assert.Equal(t, "blah blah", string(data)) - assert.Equal(t, int64(9), sz) + assert.Equal(t, 1462, len(data)) + assert.Equal(t, int64(1462), sz) _, _, err = svc.Load("abcd") assert.NotNil(t, err) } @@ -150,22 +165,22 @@ func TestFsStore_Cleanup(t *testing.T) { svc, teardown := prepareImageTest(t) defer teardown() - save := func(file string, user string, content string) (path string) { - id, err := svc.Save(file, user, strings.NewReader(content)) + save := func(file string, user string) (path string) { + id, err := svc.Save(file, user, gopherPNG()) require.NoError(t, err) img := svc.location(svc.Staging, id) data, err := ioutil.ReadFile(img) require.NoError(t, err) - require.Equal(t, content, string(data)) + assert.Equal(t, 1462, len(data)) return img } // save 3 images to staging - img1 := save("blah_ff1.png", "user1", "blah blah1") + img1 := save("blah_ff1.png", "user1") time.Sleep(100 * time.Millisecond) - img2 := save("blah_ff2.png", "user1", "blah blah2") + img2 := save("blah_ff2.png", "user1") time.Sleep(100 * time.Millisecond) - img3 := save("blah_ff3.png", "user2", "blah blah3") + img3 := save("blah_ff3.png", "user2") time.Sleep(100 * time.Millisecond) // make first image expired err := svc.Cleanup(context.Background(), time.Millisecond*300) @@ -199,7 +214,7 @@ func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) { Location: loc, Staging: staging, Partitions: 100, - MaxSize: 50, + MaxSize: 1500, } teardown = func() { diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index 112fbd81..2a3554a3 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -8,6 +8,7 @@ package image import ( "context" "io" + "net/http" "strings" "sync" "sync/atomic" @@ -16,6 +17,7 @@ import ( "github.com/PuerkitoBio/goquery" log "github.com/go-pkgz/lgr" "github.com/pkg/errors" + "github.com/rs/xid" ) // Store defines interface for saving and loading pictures. @@ -126,3 +128,14 @@ func (s *Service) Close() { } s.wg.Wait() } + +// check if file f is a valid image format, i.e. gif, png or jpeg +func isValidImage(b []byte) bool { + ct := http.DetectContentType(b) + return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" +} + +// guid makes a globally unique id +func guid() string { + return xid.New().String() +} diff --git a/backend/go.mod b/backend/go.mod index 577d345e..4305cc43 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -30,6 +30,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.8.1 github.com/rakyll/statik v0.1.3 + github.com/rs/xid v1.2.1 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/stretchr/testify v1.3.0 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 diff --git a/backend/go.sum b/backend/go.sum index ee3962c0..041faf66 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -83,6 +83,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU= github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=