mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make shorted image uids, check image type and restrict to png, gif and jpeg only #285

Browse Source
This commit is contained in:
Umputun
2019-04-13 14:27:40 -05:00
parent e182d34c2c
commit d72154b3c8
5 changed files with 64 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/store/image/fs_store.go
+14 -8
View File
@@ -1,6 +1,7 @@
package image
import (
"bytes"
"context"
"fmt"
"hash/crc64"
@@ -15,7 +16,6 @@ import (
"time"
log "github.com/go-pkgz/lgr"
"github.com/google/uuid"
"github.com/pkg/errors"
)
@@ -38,12 +38,7 @@ type FileSystem struct {
// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
uid, err := uuid.NewUUID()
if err != nil {
return "", errors.Wrap(err, "can't make image uuid")
}
id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
dst := f.location(f.Staging, id)
if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
@@ -54,20 +49,31 @@ func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id strin
if err != nil {
return "", errors.Wrapf(err, "can't make image file %s", dst)
}
lr := io.LimitReader(r, int64(f.MaxSize)+1)
written, err := io.Copy(fh, lr)
// read header first, needed it to check if data is valid png/gif/jpeg
header := make([]byte, 512)
hl, err := lr.Read(header)
if !isValidImage(header) {
return "", errors.Errorf("file %s is not in allowed format", fileName)
}
written, err := io.Copy(fh, io.MultiReader(bytes.NewReader(header[:hl]), lr)) // write header and the rest of input
if err != nil {
return "", errors.Wrapf(err, "can't write image file %s", dst)
}
if err = fh.Close(); err != nil {
return "", errors.Wrapf(err, "can't close image file %s", dst)
}
if written > int64(f.MaxSize) {
if err = os.Remove(dst); err != nil {
log.Printf("[WARN] can't remove image file %s, %v", dst, err)
}
return "", errors.Errorf("file %s is too large", fileName)
}
log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName)
return id, nil
}
backend/app/store/image/fs_store_test.go
+34 -19
View File
@@ -2,6 +2,8 @@ package image
import (
"context"
"encoding/base64"
"io"
"io/ioutil"
"math/rand"
"os"
@@ -14,11 +16,16 @@ import (
"github.com/stretchr/testify/require"
)
// gopher png for test, from https://golang.org/src/image/png/example_test.go
const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
func TestFsStore_Save(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
id, err := svc.Save("file1.png", "user1", gopherPNG())
assert.NoError(t, err)
assert.Contains(t, id, "user1/")
assert.Contains(t, id, ".png")
@@ -28,14 +35,22 @@ func TestFsStore_Save(t *testing.T) {
t.Log(img)
data, err := ioutil.ReadFile(img)
assert.NoError(t, err)
assert.Equal(t, "blah blah", string(data))
assert.Equal(t, 1462, len(data))
}
func TestFsStore_WrongFormat(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
assert.EqualError(t, err, "file file1.png is not in allowed format")
}
func TestFsStore_SaveAndCommit(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
id, err := svc.Save("file1.png", "user1", gopherPNG())
require.NoError(t, err)
err = svc.Commit(id)
require.NoError(t, err)
@@ -48,14 +63,14 @@ func TestFsStore_SaveAndCommit(t *testing.T) {
t.Log(img)
data, err := ioutil.ReadFile(img)
assert.NoError(t, err)
assert.Equal(t, "blah blah", string(data))
assert.Equal(t, 1462, len(data))
}
func TestFsStore_SaveTooLarge(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
svc.MaxSize = 5
_, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah"))
svc.MaxSize = 2000
_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
assert.Error(t, err)
assert.Contains(t, err.Error(), "is too large")
}
@@ -65,7 +80,7 @@ func TestFsStore_LoadAfterSave(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
assert.NoError(t, err)
t.Log(id)
@@ -74,8 +89,8 @@ func TestFsStore_LoadAfterSave(t *testing.T) {
defer func() { assert.NoError(t, r.Close()) }()
data, err := ioutil.ReadAll(r)
assert.NoError(t, err)
assert.Equal(t, "blah blah", string(data))
assert.Equal(t, int64(9), sz)
assert.Equal(t, 1462, len(data))
assert.Equal(t, int64(1462), sz)
_, _, err = svc.Load("abcd")
assert.NotNil(t, err)
}
@@ -85,7 +100,7 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
assert.NoError(t, err)
t.Log(id)
err = svc.Commit(id)
@@ -96,8 +111,8 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
defer func() { assert.NoError(t, r.Close()) }()
data, err := ioutil.ReadAll(r)
assert.NoError(t, err)
assert.Equal(t, "blah blah", string(data))
assert.Equal(t, int64(9), sz)
assert.Equal(t, 1462, len(data))
assert.Equal(t, int64(1462), sz)
_, _, err = svc.Load("abcd")
assert.NotNil(t, err)
}
@@ -150,22 +165,22 @@ func TestFsStore_Cleanup(t *testing.T) {
svc, teardown := prepareImageTest(t)
defer teardown()
save := func(file string, user string, content string) (path string) {
id, err := svc.Save(file, user, strings.NewReader(content))
save := func(file string, user string) (path string) {
id, err := svc.Save(file, user, gopherPNG())
require.NoError(t, err)
img := svc.location(svc.Staging, id)
data, err := ioutil.ReadFile(img)
require.NoError(t, err)
require.Equal(t, content, string(data))
assert.Equal(t, 1462, len(data))
return img
}
// save 3 images to staging
img1 := save("blah_ff1.png", "user1", "blah blah1")
img1 := save("blah_ff1.png", "user1")
time.Sleep(100 * time.Millisecond)
img2 := save("blah_ff2.png", "user1", "blah blah2")
img2 := save("blah_ff2.png", "user1")
time.Sleep(100 * time.Millisecond)
img3 := save("blah_ff3.png", "user2", "blah blah3")
img3 := save("blah_ff3.png", "user2")
time.Sleep(100 * time.Millisecond) // make first image expired
err := svc.Cleanup(context.Background(), time.Millisecond*300)
@@ -199,7 +214,7 @@ func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
Location: loc,
Staging: staging,
Partitions: 100,
MaxSize: 50,
MaxSize: 1500,
}
teardown = func() {
backend/app/store/image/image.go
+13
View File
@@ -8,6 +8,7 @@ package image
import (
"context"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
@@ -16,6 +17,7 @@ import (
"github.com/PuerkitoBio/goquery"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/rs/xid"
)
// Store defines interface for saving and loading pictures.
@@ -126,3 +128,14 @@ func (s *Service) Close() {
}
s.wg.Wait()
}
// check if file f is a valid image format, i.e. gif, png or jpeg
func isValidImage(b []byte) bool {
ct := http.DetectContentType(b)
return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg"
}
// guid makes a globally unique id
func guid() string {
return xid.New().String()
}
backend/go.mod
+1
View File
@@ -30,6 +30,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.8.1
github.com/rakyll/statik v0.1.3
github.com/rs/xid v1.2.1
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/stretchr/testify v1.3.0
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16
backend/go.sum
+2
View File
@@ -83,6 +83,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=