diff --git a/README.md b/README.md index 56580b88..1689b23f 100644 --- a/README.md +++ b/README.md @@ -115,9 +115,10 @@ _this is the recommended way to run remark42_ | cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | | cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited | | cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited | -| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, `bolt`, or `mongo` | +| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, `bolt`, or `uri` | | avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store | | avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store | +| avatar.uri | AVATAR_URI | `./var/avatars` | avatar store uri | | avatar.rsz-lmt | AVATAR_RSZ_LMT | `0` (disabled) | max image size for resizing avatars on save | | image.type | IMAGE_TYPE | `fs` | type of image storage, `fs`, `bolt`, or `mongo` | | image.max-size | IMAGE_MAX_SIZE | `5000000` | max size of image file | diff --git a/backend/_example/memory_store/go.sum b/backend/_example/memory_store/go.sum index 90b87539..4ce03db1 100644 --- a/backend/_example/memory_store/go.sum +++ b/backend/_example/memory_store/go.sum @@ -48,6 +48,7 @@ github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1 github.com/go-pkgz/auth v0.4.1/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM= github.com/go-pkgz/auth v0.7.2/go.mod h1:ibOpZYISiaOvAHe2bsKj2s3v4AkMam2WxxIFn+zhulo= github.com/go-pkgz/auth v0.8.0/go.mod h1:MBNrhig13KG0iXz/0d+30lnFUgzTxncUlaQ8suH/2p8= +github.com/go-pkgz/auth v0.8.1/go.mod h1:MBNrhig13KG0iXz/0d+30lnFUgzTxncUlaQ8suH/2p8= github.com/go-pkgz/auth/_example v0.0.0-20190722170031-705d3f732438/go.mod h1:rvZtFFkmm3p+E0CHmfUqTGKweCVg2ddsRUrEEPB1iaE= github.com/go-pkgz/auth/_example v0.0.0-20190823054154-cc9c87832e19/go.mod h1:M9HLEzuiKQQ+rwGA9bAzmUDguCJi2HyGwKkxb5ouGJU= github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM= diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index b8aa1be6..473931d1 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -132,14 +132,15 @@ type ImageGroup struct { // AvatarGroup defines options group for avatar params type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"uri" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` Bolt struct { File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"` } `group:"bolt" namespace:"bolt" env-namespace:"bolt"` - RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` + URI string `long:"uri" env:"URI" default:"./var/avatars" description:"avatar's store URI"` + RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"` } // CacheGroup defines options group for cache params @@ -470,6 +471,8 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { return nil, err } return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{}) + case "uri": + return avatar.NewStore(s.Avatar.URI) } return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type) } diff --git a/backend/go.mod b/backend/go.mod index 6b72a947..a2bee37e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -17,7 +17,7 @@ require ( github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/cors v1.0.0 github.com/go-chi/render v1.0.1 - github.com/go-pkgz/auth v0.8.0 + github.com/go-pkgz/auth v0.8.1 github.com/go-pkgz/jrpc v0.1.0 github.com/go-pkgz/lcw v0.3.1 github.com/go-pkgz/lgr v0.6.3 diff --git a/backend/go.sum b/backend/go.sum index 88ee3d91..5dc308df 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -53,8 +53,8 @@ github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VCh github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-pkgz/auth v0.4.1/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM= -github.com/go-pkgz/auth v0.8.0 h1:8m0HyTbYGA568NEBQE7CqMgBZ+ZZFwBIZaVOzOWC2uI= -github.com/go-pkgz/auth v0.8.0/go.mod h1:MBNrhig13KG0iXz/0d+30lnFUgzTxncUlaQ8suH/2p8= +github.com/go-pkgz/auth v0.8.1 h1:HBJau0LDOZ9NI5+fnrJu5azETuXLJx9ipx5uosYdK8I= +github.com/go-pkgz/auth v0.8.1/go.mod h1:MBNrhig13KG0iXz/0d+30lnFUgzTxncUlaQ8suH/2p8= github.com/go-pkgz/auth/_example v0.0.0-20190722170031-705d3f732438/go.mod h1:rvZtFFkmm3p+E0CHmfUqTGKweCVg2ddsRUrEEPB1iaE= github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM= github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8= diff --git a/backend/vendor/github.com/go-pkgz/auth/README.md b/backend/vendor/github.com/go-pkgz/auth/README.md index d3c779b8..1afccf74 100644 --- a/backend/vendor/github.com/go-pkgz/auth/README.md +++ b/backend/vendor/github.com/go-pkgz/auth/README.md @@ -34,7 +34,7 @@ Example with chi router: ```go func main() { - /// define options + // define options options := auth.Opts{ SecretReader: token.SecretFunc(func(id string) (string, error) { // secret key for JWT return "secret", nil @@ -129,7 +129,10 @@ Direct links to avatars won't survive any real-life usage if they linked from a - `avatar.GridFS` - external [GridFS](https://docs.mongodb.com/manual/core/gridfs/) (mongo db). - In case of need custom implementations of other stores can be passed in and used by `auth` library. Each store has to implement `avatar.Store` [interface](https://github.com/go-pkgz/auth/blob/master/avatar/store.go#L25). - All avatar-related setup done as a part of `auth.Opts` and needs: - - `AvatarStore` - avatar store to use, i.e. `avatar.NewLocalFS("/tmp/avatars")` + - `AvatarStore` - avatar store to use, i.e. `avatar.NewLocalFS("/tmp/avatars")` or more generic `avatar.NewStore(uri)` + - file system uri - `file:///tmp/location` or just `/tmp/location` + - boltdb - `bolt://tmp/avatars.bdb` + - mongo - `"mongodb://127.0.0.1:27017/test?ava_db=db1&ava_coll=coll1` - `AvatarRoutePath` - route prefix for direct links to proxied avatar. For example `/api/v1/avatars` will make full links like this - `http://example.com/api/v1/avatars/1234567890123.image`. The url will be stored in user's token and retrieved by middleware (see "User Info") - `AvatarResizeLimit` - size (in pixels) used to resize the avatar. Pls note - resize happens once as a part of `Put` call, i.e. on login. 0 size (default) disables resizing. diff --git a/backend/vendor/github.com/go-pkgz/auth/avatar/bolt.go b/backend/vendor/github.com/go-pkgz/auth/avatar/bolt.go index 08d26198..24b1092c 100644 --- a/backend/vendor/github.com/go-pkgz/auth/avatar/bolt.go +++ b/backend/vendor/github.com/go-pkgz/auth/avatar/bolt.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha1" "encoding/hex" + "fmt" "io" "io/ioutil" "log" @@ -126,6 +127,10 @@ func (b *BoltDB) Close() error { return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName) } +func (b *BoltDB) String() string { + return fmt.Sprintf("boltdb, path=%s", b.fileName) +} + func (b *BoltDB) sha1(data []byte, avatarID string) (id string) { h := sha1.New() if _, err := h.Write(data); err != nil { diff --git a/backend/vendor/github.com/go-pkgz/auth/avatar/gridfs.go b/backend/vendor/github.com/go-pkgz/auth/avatar/gridfs.go index 5eeb7214..04d54e73 100644 --- a/backend/vendor/github.com/go-pkgz/auth/avatar/gridfs.go +++ b/backend/vendor/github.com/go-pkgz/auth/avatar/gridfs.go @@ -2,6 +2,7 @@ package avatar import ( "bytes" + "fmt" "io" "io/ioutil" "log" @@ -116,3 +117,7 @@ func (gf *GridFS) List() (ids []string, err error) { func (gf *GridFS) Close() error { return nil } + +func (gf *GridFS) String() string { + return fmt.Sprintf("mongo (grid fs), conn=%s", gf.Connection) +} diff --git a/backend/vendor/github.com/go-pkgz/auth/avatar/localfs.go b/backend/vendor/github.com/go-pkgz/auth/avatar/localfs.go index bed7b634..7336c3a8 100644 --- a/backend/vendor/github.com/go-pkgz/auth/avatar/localfs.go +++ b/backend/vendor/github.com/go-pkgz/auth/avatar/localfs.go @@ -108,6 +108,10 @@ func (fs *LocalFS) Close() error { return nil } +func (fs *LocalFS) String() string { + return fmt.Sprintf("localfs, path=%s", fs.storePath) +} + // get location (directory) for user id by adding partition to final path in order to keep files // in different subdirectories and avoid too many files in a single place. // the end result is a full path like this - /tmp/avatars.test/92 diff --git a/backend/vendor/github.com/go-pkgz/auth/avatar/store.go b/backend/vendor/github.com/go-pkgz/auth/avatar/store.go index 8aec56e3..e6c58740 100644 --- a/backend/vendor/github.com/go-pkgz/auth/avatar/store.go +++ b/backend/vendor/github.com/go-pkgz/auth/avatar/store.go @@ -4,15 +4,21 @@ package avatar import ( "crypto/sha1" + "fmt" _ "image/gif" // initializing packages for supporting GIF _ "image/jpeg" // initializing packages for supporting JPEG. _ "image/png" // initializing packages for supporting PNG. "io" "log" + "net/url" "regexp" "strings" + "time" + bolt "github.com/coreos/bbolt" "github.com/go-pkgz/auth/token" + "github.com/go-pkgz/mongo" + "github.com/pkg/errors" ) // imgSfx for avatars @@ -22,6 +28,7 @@ var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`) // Store defines interface to store and and load avatars type Store interface { + fmt.Stringer Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader ID(avatarID string) (id string) // unique id of stored avatar's data @@ -30,6 +37,30 @@ type Store interface { Close() error // close store } +// NewStore provides factory for all supported stores making the one +// based on uri protocol. Default (no protocol) is file-system +func NewStore(uri string) (Store, error) { + switch { + case strings.HasPrefix(uri, "file://"): + return NewLocalFS(strings.TrimPrefix(uri, "file://")), nil + case !strings.Contains(uri, "://"): + return NewLocalFS(uri), nil + case strings.HasPrefix(uri, "mongodb://"): + db, coll, u, err := parseExtMongoURI(uri) + if err != nil { + return nil, errors.Wrapf(err, "can't parse mongo store uri %s", uri) + } + mg, err := mongo.NewServerWithURL(u, time.Second) + if err != nil { + return nil, errors.Wrap(err, "failed to make mongo server") + } + return NewGridFS(mongo.NewConnection(mg, db, coll)), nil + case strings.HasPrefix(uri, "bolt://"): + return NewBoltDB(strings.TrimPrefix(uri, "bolt://"), bolt.Options{}) + } + return nil, errors.Errorf("can't parse store url %s", uri) +} + // Migrate avatars between stores func Migrate(dst, src Store) (int, error) { ids, err := src.List() @@ -59,3 +90,26 @@ func encodeID(id string) string { } return token.HashID(sha1.New(), id) } + +// parseExtMongoURI extracts extra params ava_db and ava_coll and remove +// from the url. Input example: mongodb://user:password@127.0.0.1:27017/test?ssl=true&ava_db=db1&ava_coll=coll1 +func parseExtMongoURI(uri string) (db, collection, cleanURI string, err error) { + + db, collection = "test", "avatars_fs" + u, err := url.Parse(uri) + if err != nil { + return "", "", "", err + } + if val := u.Query().Get("ava_db"); val != "" { + db = val + } + if val := u.Query().Get("ava_coll"); val != "" { + collection = val + } + + q := u.Query() + q.Del("ava_db") + q.Del("ava_coll") + u.RawQuery = q.Encode() + return db, collection, u.String(), nil +} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 0294cd54..3eb350cf 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -33,7 +33,7 @@ github.com/go-chi/chi/middleware github.com/go-chi/cors # github.com/go-chi/render v1.0.1 github.com/go-chi/render -# github.com/go-pkgz/auth v0.8.0 +# github.com/go-pkgz/auth v0.8.1 github.com/go-pkgz/auth github.com/go-pkgz/auth/avatar github.com/go-pkgz/auth/provider