diff --git a/.gitignore b/.gitignore index 0f74b453..ec8cdf79 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ debug.test .mongo remark42 /bin/ -/backend/var/ \ No newline at end of file +/backend/var/ diff --git a/Dockerfile b/Dockerfile index 8fec5895..a52f40c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,9 @@ WORKDIR /build/backend RUN \ if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ cd app && \ - if [ -z "$SKIP_BACKEND_TEST" ] ; then go test -mod=vendor -covermode=count -coverprofile=/profile.cov ./... ; \ + if [ -z "$SKIP_BACKEND_TEST" ] ; then \ + go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \ + cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \ else echo "skip backend test" ; fi RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts @@ -37,10 +39,10 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts # linters RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ - golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \ - --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \ - --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \ - --enable=ineffassign --enable=varcheck ./... ; \ + golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \ + --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \ + --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \ + --enable=ineffassign --enable=varcheck ./... ; \ else echo "skip backend linters" ; fi # submit coverage to coverals if COVERALLS_TOKEN in env @@ -50,8 +52,7 @@ RUN if [ -z "$COVERALLS_TOKEN" ] ; then \ # if DRONE presented use DRONE_* git env to make version RUN \ - if [ -z "$DRONE" ] ; then \ - echo "runs outside of drone" && version="local"; \ + if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \ else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \ echo "version=$version" && \ go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app diff --git a/README.md b/README.md index 82e3f92d..b449f9bc 100644 --- a/README.md +++ b/README.md @@ -86,63 +86,68 @@ _this is the recommended way to run remark42_ #### Parameters -| Command line | Environment | Default | Description | -| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ | -| url | REMARK_URL | | url to remark42 server, _required_ | -| secret | SECRET | | secret key, _required_ | -| site | SITE | `remark` | site name(s), _multi_ | -| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` | -| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory | -| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout | -| mongo.url | MONGO_URL | | mongo url for all stores using mongodb | -| mongo.db | MONGO_DB | | mongo database | -| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ | -| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email | -| backup | BACKUP_PATH | `./var/backup` | backups location | -| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep | -| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | -| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited | -| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited | -| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, 'bolt`, or `mongo` | -| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store | -| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store | -| avatar.rsz-lmt | AVATAR_RSZ_LMT | `0` (disabled) | max image size for resizing avatars on save | -| auth.ttl.jwt | AUTH_TTL_JWT | `5m` | jwt TTL | -| auth.ttl.cookie | AUTH_TTL_COOKIE | `200h` | cookie TTL | -| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID | -| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret | -| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID | -| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret | -| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID | -| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret | -| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID | -| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret | -| auth.dev | AUTH_DEV | `false` | local oauth2 server, development mode only | -| auth.anon | AUTH_ANON | `false` | enable anonymous login | -| notify.type | NOTIFY_TYPE | none | type of notification (none or telegram) | -| notify.queue | NOTIFY_QUEUE | `100` | size of notification queue | -| notify.telegram.token | NOTIFY_TELEGRAM_TOKEN | | telegram token | -| notify.telegram.chan | NOTIFY_TELEGRAM_CHAN | | telegram channel | -| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s` | telegram timeout | -| ssl.type | SSL_TYPE | none | `none`-http, `static`-https, `auto`-https + le | -| ssl.port | SSL_PORT | `8443` | port for https server | -| ssl.cert | SSL_CERT | | path to cert.pem file | -| ssl.key | SSL_KEY | | path to key.pem file | -| ssl.acme-location | SSL_ACME_LOCATION | `./var/acme` | dir where obtained le-certs will be stored | -| ssl.acme-email | SSL_ACME_EMAIL | | admin email for receiving notifications from LE | -| max-comment | MAX_COMMENT_SIZE | `2048` | comment's size limit | -| max-votes | MAX_VOTES | `-1` | votes limit per comment, `-1` - unlimited | -| low-score | LOW_SCORE | `-5` | low score threshold | -| positive-score | POSITIVE_SCORE | `false` | enable positive score only | -| critical-score | CRITICAL_SCORE | `-10` | critical score threshold | -| positive-score | POSITIVE_SCORE | `false` | restricts comment's score to be only positive | -| restricted-words | RESTRICTED_WORDS | | words banned in comments (can use `*`), _multi_ | -| edit-time | EDIT_TIME | `5m` | edit window | -| read-age | READONLY_AGE | | read-only age of comments, days | -| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images | -| update-limit | UPDATE_LIMIT | `0.5` | updates/sec limit | -| admin-passwd | ADMIN_PASSWD | none (disabled) | password for `admin` basic auth | -| dbg | DEBUG | `false` | debug mode | +| Command line | Environment | Default | Description | +| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ | +| url | REMARK_URL | | url to remark42 server, _required_ | +| secret | SECRET | | secret key, _required_ | +| site | SITE | `remark` | site name(s), _multi_ | +| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` | +| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory | +| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout | +| mongo.url | MONGO_URL | | mongo url for all stores using mongodb | +| mongo.db | MONGO_DB | | mongo database | +| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ | +| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email | +| backup | BACKUP_PATH | `./var/backup` | backups location | +| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep | +| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited | +| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited | +| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited | +| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, 'bolt`, or `mongo` | +| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store | +| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store | +| avatar.rsz-lmt | AVATAR_RSZ_LMT | `0` (disabled) | max image size for resizing avatars on save | +| image.type | IMAGE_TYPE | `fs` | type of image storage, `fs`, 'bolt`, or `mongo` | +| image.max-size | IMAGE_MAX_SIZE | `5000000` | max size of image file | +| image.fs.path | IMAGE_FS_PATH | `./var/pictures` | permanent location of images | +| image.fs.staging | IMAGE_FS_STAGING | `./var/pictures.staging` | staging location of images | +| image.fs.partitions | IMAGE_FS_PARTITIONS | `100` | number of image partitions | +| auth.ttl.jwt | AUTH_TTL_JWT | `5m` | jwt TTL | +| auth.ttl.cookie | AUTH_TTL_COOKIE | `200h` | cookie TTL | +| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID | +| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret | +| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID | +| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret | +| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID | +| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret | +| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID | +| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret | +| auth.dev | AUTH_DEV | `false` | local oauth2 server, development mode only | +| auth.anon | AUTH_ANON | `false` | enable anonymous login | +| notify.type | NOTIFY_TYPE | none | type of notification (none or telegram) | +| notify.queue | NOTIFY_QUEUE | `100` | size of notification queue | +| notify.telegram.token | NOTIFY_TELEGRAM_TOKEN | | telegram token | +| notify.telegram.chan | NOTIFY_TELEGRAM_CHAN | | telegram channel | +| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s` | telegram timeout | +| ssl.type | SSL_TYPE | none | `none`-http, `static`-https, `auto`-https + le | +| ssl.port | SSL_PORT | `8443` | port for https server | +| ssl.cert | SSL_CERT | | path to cert.pem file | +| ssl.key | SSL_KEY | | path to key.pem file | +| ssl.acme-location | SSL_ACME_LOCATION | `./var/acme` | dir where obtained le-certs will be stored | +| ssl.acme-email | SSL_ACME_EMAIL | | admin email for receiving notifications from LE | +| max-comment | MAX_COMMENT_SIZE | `2048` | comment's size limit | +| max-votes | MAX_VOTES | `-1` | votes limit per comment, `-1` - unlimited | +| low-score | LOW_SCORE | `-5` | low score threshold | +| positive-score | POSITIVE_SCORE | `false` | enable positive score only | +| critical-score | CRITICAL_SCORE | `-10` | critical score threshold | +| positive-score | POSITIVE_SCORE | `false` | restricts comment's score to be only positive | +| restricted-words | RESTRICTED_WORDS | | words banned in comments (can use `*`), _multi_ | +| edit-time | EDIT_TIME | `5m` | edit window | +| read-age | READONLY_AGE | | read-only age of comments, days | +| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images | +| update-limit | UPDATE_LIMIT | `0.5` | updates/sec limit | +| admin-passwd | ADMIN_PASSWD | none (disabled) | password for `admin` basic auth | +| dbg | DEBUG | `false` | debug mode | * command line parameters are long form `--=value`, i.e. `--site=https://demo.remark42.com` * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...` @@ -602,6 +607,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i * `GET /api/v1/rss/site?site=site-id` - rss feed for given site * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments +### Images management + +* `GET /api/v1/picture/{user}/{id}` - load stored image +* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_ + +_returned id should be appended to load image url on caller side_ + ### Admin * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`. diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index b93d6b32..5c7e9a92 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -31,6 +31,7 @@ type AvatarMigrator interface { type avatarMigrator struct{} +// Migrate from one avatar store to another. Can be used to convert between stores func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) { return avatar.Migrate(dst, src) } diff --git a/backend/app/cmd/cleanup.go b/backend/app/cmd/cleanup.go index c7b3e3e5..bbd29417 100644 --- a/backend/app/cmd/cleanup.go +++ b/backend/app/cmd/cleanup.go @@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error) Info store.PostInfo `json:"info,omitempty"` }{} - if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil { + if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil { return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL) } return commentsWithInfo.Comments, nil diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 6579398a..5b88d957 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -15,7 +15,7 @@ import ( bolt "github.com/coreos/bbolt" log "github.com/go-pkgz/lgr" - auth_cache "github.com/patrickmn/go-cache" + authcache "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/go-pkgz/auth" @@ -32,6 +32,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -43,6 +44,7 @@ type ServerCommand struct { Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"` Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"` + Image ImageGroup `group:"image" namespace:"image" env-namespace:"IMAGE"` SSL SSLGroup `group:"ssl" namespace:"ssl" env-namespace:"SSL"` Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","` @@ -93,6 +95,20 @@ type StoreGroup struct { } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` } +// ImageGroup defines options group for store pictures +type ImageGroup struct { + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` + FS struct { + Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"` + Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"` + Partitions int `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"` + } `group:"fs" namespace:"fs" env-namespace:"FS"` + Bolt struct { + File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"` + } `group:"bolt" namespace:"bolt" env-namespace:"bolt"` + MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"` +} + // AvatarGroup defines options group for avatar params type AvatarGroup struct { Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` @@ -162,6 +178,7 @@ type serverApp struct { dataService *service.DataStore avatarStore avatar.Store notifyService *notify.Service + imageService *image.Service terminated chan struct{} } @@ -182,6 +199,7 @@ func (s *ServerCommand) Execute(args []string) error { app, err := s.newServerApp() if err != nil { log.Printf("[PANIC] failed to setup application, %+v", err) + return err } if err = app.run(ctx); err != nil { log.Printf("[ERROR] remark terminated with error %+v", err) @@ -214,6 +232,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { return nil, errors.Wrap(err, "failed to make admin store") } + imageService, err := s.makePicturesStore() + if err != nil { + return nil, errors.Wrap(err, "failed to make pictures store") + } + dataService := &service.DataStore{ Interface: storeEngine, EditDuration: s.EditDuration, @@ -221,6 +244,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { MaxCommentSize: s.MaxCommentSize, MaxVotes: s.MaxVotes, PositiveScore: s.PositiveScore, + ImageService: imageService, TitleExtractor: service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}), RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}), } @@ -276,15 +300,16 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { NotifyService: notifyService, SSLConfig: sslConfig, UpdateLimiter: s.UpdateLimit, + ImageService: imageService, } srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore var devAuth *provider.DevAuthServer if s.Auth.Dev { - da, err := authenticator.DevAuth() - if err != nil { - return nil, errors.Wrap(err, "can't make dev oauth2 server") + da, errDevAuth := authenticator.DevAuth() + if errDevAuth != nil { + return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server") } devAuth = da } @@ -298,6 +323,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { dataService: dataService, avatarStore: avatarStore, notifyService: notifyService, + imageService: imageService, terminated: make(chan struct{}), }, nil } @@ -323,12 +349,17 @@ func (a *serverApp) run(ctx context.Context) error { log.Printf("[WARN] failed to close avatar store, %s", e) } a.notifyService.Close() + a.imageService.Close() log.Print("[INFO] shutdown completed") }() + a.activateBackup(ctx) // runs in goroutine for each site if a.Auth.Dev { go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084 } + + go a.imageService.Cleanup(ctx) // pictures cleanup for staging images + a.restSrv.Run(a.Port) close(a.terminated) return nil @@ -405,6 +436,25 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type) } +func (s *ServerCommand) makePicturesStore() (*image.Service, error) { + switch s.Image.Type { + case "fs": + if err := makeDirs(s.Image.FS.Path); err != nil { + return nil, err + } + return &image.Service{ + Store: &image.FileSystem{ + Location: s.Image.FS.Path, + Staging: s.Image.FS.Staging, + Partitions: s.Image.FS.Partitions, + MaxSize: s.Image.MaxSize, + }, + TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging + }, nil + } + return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type) +} + func (s *ServerCommand) makeAdminStore() (admin.Store, error) { log.Printf("[INFO] make admin store, type=%s", s.Admin.Type) @@ -585,17 +635,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto // authRefreshCache used by authenticator to minimize repeatable token refreshes type authRefreshCache struct { - *auth_cache.Cache + *authcache.Cache } func newAuthRefreshCache() *authRefreshCache { - return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)} + return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)} } +// Get implements cache getter with key converted to string func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) { return c.Cache.Get(key.(string)) } +// Set implements cache setter with key converted to string func (c *authRefreshCache) Set(key, value interface{}) { - c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration) + c.Cache.Set(key.(string), value, authcache.DefaultExpiration) } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 9e0f2309..6b56319e 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -24,7 +24,7 @@ import ( ) func TestServerApp(t *testing.T) { - app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand { + app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand { o.Port = 18080 return o }) diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go index bbe0b260..14a5dc89 100644 --- a/backend/app/migrator/disqus.go +++ b/backend/app/migrator/disqus.go @@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { if se.Name.Local == "thread" { stats.inpThreads++ thread := disqusThread{} - if err := decoder.DecodeElement(&thread, &se); err != nil { + if err = decoder.DecodeElement(&thread, &se); err != nil { log.Printf("[WARN] can't decode disqus thread, %s", err) stats.failedThreads++ continue @@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) { if se.Name.Local == "post" { stats.inpComments++ comment := disqusComment{} - if err := decoder.DecodeElement(&comment, &se); err != nil { + if err = decoder.DecodeElement(&comment, &se); err != nil { log.Printf("[WARN] can't decode disqus comment, %s", err) stats.failedPosts++ continue diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go index 99cf9d34..40db2a17 100644 --- a/backend/app/migrator/native.go +++ b/backend/app/migrator/native.go @@ -15,7 +15,7 @@ import ( "github.com/umputun/remark/backend/app/store/service" ) -const natvieVersion = 1 +const nativeVersion = 1 const defaultConcurrent = 8 // Native implements exporter and importer for internal store format @@ -50,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) { for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction topic := topics[i] comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time") - if err != nil { + if e != nil { return commentsCount, e } @@ -75,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) { // exportMeta appends user and post metas to exported stream func (n *Native) exportMeta(siteID string, w io.Writer) (err error) { - m := meta{Version: natvieVersion} + m := meta{Version: nativeVersion} m.Users, m.Posts, err = n.DataStore.Metas(siteID) if err != nil { return errors.Wrap(err, "can't get meta") } - if err := json.NewEncoder(w).Encode(m); err != nil { + if err = json.NewEncoder(w).Encode(m); err != nil { return errors.Wrap(err, "can't encode meta") } return nil @@ -96,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID) } - if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version + if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version return 0, errors.Errorf("unexpected import file version %d", m.Version) } @@ -134,9 +134,9 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) { log.Printf("[WARN] can't write %+v to store, %s", comment, e) return } - n := atomic.AddInt64(&comments, 1) - if n%1000 == 0 { - log.Printf("[DEBUG] imported %d comments", n) + num := atomic.AddInt64(&comments, 1) + if num%1000 == 0 { + log.Printf("[DEBUG] imported %d comments", num) } }) diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go index 90dce39b..c1199ea7 100644 --- a/backend/app/migrator/wordpress.go +++ b/backend/app/migrator/wordpress.go @@ -39,6 +39,7 @@ type wpTime struct { time time.Time } +// UnmarshalXML decoding xml with time in WP format func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var v string if err := d.DecodeElement(&v, &start); err != nil { @@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment { if el.Name.Local == "item" { stats.inpItems++ item := wpItem{} - if err := decoder.DecodeElement(&item, &el); err != nil { + if err = decoder.DecodeElement(&item, &el); err != nil { log.Printf("[WARN] Can't decode item, %s", err) stats.failedItems++ continue diff --git a/backend/app/notify/notify.go b/backend/app/notify/notify.go index 67b1b7bb..886fabad 100644 --- a/backend/app/notify/notify.go +++ b/backend/app/notify/notify.go @@ -29,10 +29,11 @@ type Destination interface { Send(ctx context.Context, req request) error } -// Store defines the minimal interface accessing stored commens used by notifier +// Store defines the minimal interface accessing stored comments used by notifier type Store interface { Get(locator store.Locator, id string) (store.Comment, error) } + type request struct { comment store.Comment parent store.Comment diff --git a/backend/app/notify/telegram_test.go b/backend/app/notify/telegram_test.go index 8c429d5b..35dd2a21 100644 --- a/backend/app/notify/telegram_test.go +++ b/backend/app/notify/telegram_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" ) @@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) { tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/") assert.NotNil(t, err, "should failed") err = tb.Send(context.TODO(), request{comment: c, parent: cp}) + require.NotNil(t, err) assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg") assert.Equal(t, "telegram: @remark_test", tb.String()) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 6f634e29..cba6472a 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -115,14 +115,14 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { return } - if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil { + if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess) return } if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil { - avatartStore := a.authenticator.AvatarProxy().Store - if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil { + avatarStore := a.authenticator.AvatarProxy().Store + if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal) return } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index ccad9e37..7844217e 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -30,6 +30,7 @@ import ( "github.com/umputun/remark/backend/app/rest" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -44,6 +45,7 @@ type Rest struct { CommentFormatter *store.CommentFormatter Migrator *Migrator NotifyService *notify.Service + ImageService *image.Service WebRoot string RemarkURL string @@ -219,6 +221,7 @@ func (s *Rest) routes() chi.Router { ropen.Get("/config", s.configCtrl) ropen.Post("/preview", s.previewCommentCtrl) ropen.Get("/info", s.infoCtrl) + ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl) ropen.Mount("/rss", s.rssRoutes()) ropen.Mount("/img", s.ImageProxy.Routes()) @@ -253,12 +256,25 @@ func (s *Rest) routes() chi.Router { rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl) rauth.Post("/deleteme", s.deleteMeCtrl) }) + + rapi.Group(func(rauth chi.Router) { + lmt := 10.0 + if s.UpdateLimiter > 0 { + lmt = s.UpdateLimiter + } + rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil))) + rauth.Use(authMiddleware.Auth) + rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler) + rauth.Post("/picture", s.savePictureCtrl) + }) + }) // respond to /robots.txt with the list of allowed paths router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))). Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) { - allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"} + allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", + "/img", "/avatar", "/picture"} for i := range allowed { allowed[i] = "Allow: /api/v1" + allowed[i] } diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index add6fd9d..b7063656 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -228,14 +228,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) { // get comments in 100 in each paginated request for i := 0; i < 100; i++ { - comments, err := s.DataService.User(siteID, user.ID, 100, i*100) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments", rest.ErrInternal) + comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100) + if errUser != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal) return } - b, err := json.Marshal(comments) - if err != nil { - rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments", rest.ErrInternal) + b, errUser := json.Marshal(comments) + if errUser != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal) return } @@ -285,6 +285,31 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link}) } +// POST /image - save image with form request +func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) { + user := rest.MustGetUserInfo(r) + + if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode) + return + } + + file, header, err := r.FormFile("file") + if err != nil { + rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal) + return + } + defer func() { _ = file.Close() }() + + id, err := s.ImageService.Save(header.Filename, user.ID, file) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal) + return + } + + render.JSON(w, r, R.JSON{"id": id}) +} + func (s *Rest) isReadOnly(locator store.Locator) bool { if s.ReadOnlyAge > 0 { // check RO by age diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index 6131bfb0..e43d7ff1 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -1,20 +1,26 @@ package api import ( + "bytes" "compress/gzip" "encoding/json" "fmt" + "io" "io/ioutil" + "mime/multipart" "net/http" + "os" "strings" "testing" "time" + "github.com/go-pkgz/lgr" R "github.com/go-pkgz/rest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/image" ) func TestRest_Create(t *testing.T) { @@ -488,3 +494,141 @@ func TestRest_DeleteMe(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 401, resp.StatusCode) } + +func TestRest_SavePictureCtrl(t *testing.T) { + ts, _, teardown := startupT(t) + defer teardown() + + // save picture + savePic := func(name string) (id string) { + r := strings.NewReader("file content 123") + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, err := bodyWriter.CreateFormFile("file", name) + require.NoError(t, err) + _, err = io.Copy(fileWriter, r) + require.NoError(t, err) + contentType := bodyWriter.FormDataContentType() + require.NoError(t, bodyWriter.Close()) + + client := http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf) + require.NoError(t, err) + req.Header.Add("Content-Type", contentType) + req.Header.Add("X-JWT", devToken) + resp, err := client.Do(req) + assert.Nil(t, err) + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + + m := map[string]string{} + err = json.Unmarshal(body, &m) + assert.NoError(t, err) + assert.True(t, m["id"] != "") + return m["id"] + } + + id := savePic("picture.png") + resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + assert.Equal(t, "file content 123", string(body)) + assert.Equal(t, "image/png", resp.Header.Get("Content-Type")) + + id = savePic("picture.gif") + resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "image/gif", resp.Header.Get("Content-Type")) + + id = savePic("picture.jpg") + resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type")) + + id = savePic("picture.blah") + resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "image/*", resp.Header.Get("Content-Type")) + + resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL)) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) +} + +func TestRest_CreateWithPictures(t *testing.T) { + ts, svc, teardown := startupT(t) + defer func() { + teardown() + os.RemoveAll("/tmp/remark42") + }() + lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc) + + svc.ImageService = &image.Service{ + Store: &image.FileSystem{ + Staging: "/tmp/remark42/images.staging", + Location: "/tmp/remark42/images", + MaxSize: 1000, + }, + TTL: time.Millisecond * 100, + } + svc.DataService.EditDuration = time.Millisecond * 100 + svc.DataService.ImageService = svc.ImageService + + uploadPicture := func(file, content string) (id string) { + r := strings.NewReader(content) + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, err := bodyWriter.CreateFormFile("file", file) + require.NoError(t, err) + _, err = io.Copy(fileWriter, r) + require.NoError(t, err) + contentType := bodyWriter.FormDataContentType() + require.NoError(t, bodyWriter.Close()) + client := http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf) + require.NoError(t, err) + req.Header.Add("Content-Type", contentType) + req.Header.Add("X-JWT", devToken) + resp, err := client.Do(req) + assert.Nil(t, err) + assert.Equal(t, 200, resp.StatusCode) + + body, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + m := map[string]string{} + err = json.Unmarshal(body, &m) + assert.NoError(t, err) + assert.Contains(t, m["id"], ".png") + return m["id"] + } + + id1 := uploadPicture("pic1.png", "file content 123") + id2 := uploadPicture("pic2.png", "file content 12345") + id3 := uploadPicture("pic3.png", "file content xyz12365789") + + text := fmt.Sprintf(`text 123 ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3) + body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text) + + resp, err := post(t, ts.URL+"/api/v1/comment", body) + assert.Nil(t, err) + b, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + require.Equal(t, http.StatusCreated, resp.StatusCode, string(b)) + + _, err = os.Stat("/tmp/remark42/images/" + id1) + assert.NotNil(t, err, "not moved from staging yet") + + time.Sleep(300 * time.Millisecond) + _, err = os.Stat("/tmp/remark42/images/" + id1) + assert.NoError(t, err, "moved from staging") + _, err = os.Stat("/tmp/remark42/images/" + id2) + assert.NoError(t, err, "moved from staging") + _, err = os.Stat("/tmp/remark42/images/" + id3) + assert.NoError(t, err, "moved from staging") +} diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go index 40c8e99e..3657bacd 100644 --- a/backend/app/rest/api/rest_public.go +++ b/backend/app/rest/api/rest_public.go @@ -1,8 +1,9 @@ package api import ( - "crypto/sha1" //nolint + "crypto/sha1" // nolint "encoding/base64" + "io" "net/http" "strconv" "strings" @@ -274,12 +275,9 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) { // key could be long for multiple posts, make it sha1 k := URLKey(r) + strings.Join(posts, ",") - hasher := sha1.New() //nolint - if _, err := hasher.Write([]byte(k)); err != nil { - rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls", rest.ErrInternal) - return - } - sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) + h := sha1.Sum([]byte(k)) //nolint + sha := base64.URLEncoding.EncodeToString(h[:]) + key := cache.NewKey(siteID).ID(sha).Scopes(siteID) data, err := s.Cache.Get(key, func() ([]byte, error) { counts, e := s.DataService.Counts(siteID, posts) @@ -330,3 +328,46 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[WARN] can't render posts lits for site %s", siteID) } } + +// GET /picture/{user}/{id} - get picture +func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) { + + imgContentType := func(img string) string { + img = strings.ToLower(img) + switch { + case strings.HasSuffix(img, ".png"): + return "image/png" + case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"): + return "image/jpeg" + case strings.HasSuffix(img, ".gif"): + return "image/gif" + } + return "image/*" + } + + id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id") + imgRdr, size, err := s.ImageService.Load(id) + if err != nil { + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound) + return + } + // enforce client-side caching + etag := `"` + id + `"` + w.Header().Set("Etag", etag) + w.Header().Set("Cache-Control", "max-age=604800") // 7 days + if match := r.Header.Get("If-None-Match"); match != "" { + if strings.Contains(match, etag) { + w.WriteHeader(http.StatusNotModified) + return + } + } + + defer imgRdr.Close() + + w.Header().Set("Content-Type", imgContentType(id)) + w.Header().Set("Content-Length", strconv.Itoa(int(size))) + w.WriteHeader(http.StatusOK) + if _, err = io.Copy(w, imgRdr); err != nil { + log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err) + } +} diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go index 4285b384..e3343a8f 100644 --- a/backend/app/rest/api/rest_public_test.go +++ b/backend/app/rest/api/rest_public_test.go @@ -377,6 +377,37 @@ func TestRest_List(t *testing.T) { assert.Equal(t, 3, pi[1].Count) } +func TestRest_ListWithSkipAndLimit(t *testing.T) { + ts, _, teardown := startupT(t) + defer teardown() + + c1 := store.Comment{Text: "test test #1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}} + c2 := store.Comment{Text: "test test #2", ParentID: "p1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}} + c3 := store.Comment{Text: "test test #3", ParentID: "p1", + Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}} + + addComment(t, c1, ts) + addComment(t, c1, ts) + addComment(t, c1, ts) + addComment(t, c2, ts) + addComment(t, c2, ts) + addComment(t, c3, ts) + addComment(t, c3, ts) + + body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2") + assert.Equal(t, 200, code) + pi := []store.PostInfo{} + err := json.Unmarshal([]byte(body), &pi) + assert.Nil(t, err) + require.Equal(t, 2, len(pi)) + assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL) + assert.Equal(t, 2, pi[0].Count) + assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL) + assert.Equal(t, 3, pi[1].Count) +} + func TestRest_Config(t *testing.T) { ts, _, teardown := startupT(t) defer teardown() @@ -442,5 +473,5 @@ func TestRest_Robots(t *testing.T) { assert.Equal(t, 200, code) assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+ "Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+ - "Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body)) + "Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body)) } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index d21b1c6c..ef5108d7 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -29,6 +29,7 @@ import ( "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -203,6 +204,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { os.Remove(testDb) os.Remove(testHTML) os.RemoveAll("/tmp/ava-remark42") + os.RemoveAll("/tmp/pics-remark42") b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) @@ -232,7 +234,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { Cache: memCache, WebRoot: "/tmp", RemarkURL: "https://demo.remark42.com", - + ImageService: &image.Service{ + Store: &image.FileSystem{ + Location: "/tmp/pics-remark42", + Partitions: 100, + MaxSize: 10000, + }, + TTL: time.Millisecond * 100, + }, ImageProxy: &proxy.Image{}, ReadOnlyAge: 10, CommentFormatter: store.NewCommentFormatter(&proxy.Image{}), @@ -258,6 +267,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { os.Remove(testDb) os.Remove(testHTML) os.RemoveAll("/tmp/ava-remark42") + os.RemoveAll("/tmp/pics-remark42") } return ts, srv, teardown diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go index 1573befc..1b7107d2 100644 --- a/backend/app/rest/api/rss.go +++ b/backend/app/rest/api/rss.go @@ -57,7 +57,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.WriteHeader(http.StatusOK) - if _, err := w.Write(data); err != nil { + if _, err = w.Write(data); err != nil { log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err) } } @@ -89,7 +89,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.WriteHeader(http.StatusOK) - if _, err := w.Write(data); err != nil { + if _, err = w.Write(data); err != nil { log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err) } } @@ -141,7 +141,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.WriteHeader(http.StatusOK) - if _, err := w.Write(data); err != nil { + if _, err = w.Write(data); err != nil { log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err) } } diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go index 9076f874..22a62893 100644 --- a/backend/app/store/engine/bolt_accessor.go +++ b/backend/app/store/engine/bolt_accessor.go @@ -155,7 +155,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C return bucket.ForEach(func(k, v []byte) error { comment := store.Comment{} - if e := json.Unmarshal(v, &comment); e != nil { + if e = json.Unmarshal(v, &comment); e != nil { return errors.Wrap(e, "failed to unmarshal") } comments = append(comments, comment) @@ -195,7 +195,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err } comment := store.Comment{} - if e := b.load(postBkt, []byte(commentID), &comment); e != nil { + if e = b.load(postBkt, []byte(commentID), &comment); e != nil { log.Printf("[WARN] can't load comment for %s from store %s", commentID, url) continue } @@ -335,11 +335,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store. // retrieve comments for refs for _, v := range commentRefs { - url, commentID, e := b.parseRef([]byte(v)) - if e != nil { - return comments, errors.Wrapf(e, "can't parse reference %s", v) + url, commentID, errParse := b.parseRef([]byte(v)) + if errParse != nil { + return comments, errors.Wrapf(errParse, "can't parse reference %s", v) } - if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil { + if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil { comments = append(comments, c) } } diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go index 579d34db..429ac991 100644 --- a/backend/app/store/engine/bolt_admin.go +++ b/backend/app/store/engine/bolt_admin.go @@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele } comment := store.Comment{} - if err := b.load(postBkt, []byte(commentID), &comment); err != nil { + if err = b.load(postBkt, []byte(commentID), &comment); err != nil { return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) } // set deleted status and clear fields comment.SetDeleted(mode) - if err := b.save(postBkt, []byte(commentID), comment); err != nil { + if err = b.save(postBkt, []byte(commentID), comment); err != nil { return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) } // delete from "last" bucket lastBkt := tx.Bucket([]byte(lastBucketName)) - if err := lastBkt.Delete([]byte(commentID)); err != nil { + if err = lastBkt.Delete([]byte(commentID)); err != nil { return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) } @@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { return nil } - until, err := time.Parse(tsNano, string(val)) - if err != nil { + until, e := time.Parse(tsNano, string(val)) + if e != nil { blocked = false return nil } @@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { err = bdb.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(blocksBucketName)) return bucket.ForEach(func(k []byte, v []byte) error { - ts, e := time.ParseInLocation(tsNano, string(v), time.Local) - if e != nil { - return errors.Wrap(e, "can't parse block ts") + ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local) + if errParse != nil { + return errors.Wrap(errParse, "can't parse block ts") } if time.Now().Before(ts) { // get user name from comment user section userName := "" - userComments, e := b.User(siteID, string(k), 1, 0) - if e == nil && len(userComments) > 0 { + userComments, errUser := b.User(siteID, string(k), 1, 0) + if errUser == nil && len(userComments) > 0 { userName = userComments[0].User.Name } users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts}) diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go index 587e7bda..d36846d2 100644 --- a/backend/app/store/engine/mongo.go +++ b/backend/app/store/engine/mongo.go @@ -230,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) { if err != nil { return nil, err } - for _, m := range metas { - ids = append(ids, m.ID) + for _, meta := range metas { + ids = append(ids, meta.ID) } return ids, nil } diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go new file mode 100644 index 00000000..79049635 --- /dev/null +++ b/backend/app/store/image/fs_store.go @@ -0,0 +1,162 @@ +package image + +import ( + "context" + "fmt" + "hash/crc64" + "io" + "math" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + log "github.com/go-pkgz/lgr" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size. +type FileSystem struct { + Location string + Staging string + MaxSize int + Partitions int + + crc struct { + *crc64.Table + sync.Once + mask string + divider uint64 + } +} + +// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext +// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png +func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) { + + uid, err := uuid.NewUUID() + if err != nil { + return "", errors.Wrap(err, "can't make image uuid") + } + + id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext + dst := f.location(f.Staging, id) + + if err = os.MkdirAll(path.Dir(dst), 0700); err != nil { + return "", errors.Wrap(err, "can't make image directory") + } + + fh, err := os.Create(dst) + if err != nil { + return "", errors.Wrapf(err, "can't make image file %s", dst) + } + lr := io.LimitReader(r, int64(f.MaxSize)+1) + written, err := io.Copy(fh, lr) + if err != nil { + return "", errors.Wrapf(err, "can't write image file %s", dst) + } + if err = fh.Close(); err != nil { + return "", errors.Wrapf(err, "can't close image file %s", dst) + } + if written > int64(f.MaxSize) { + if err = os.Remove(dst); err != nil { + log.Printf("[WARN] can't remove image file %s, %v", dst, err) + } + return "", errors.Errorf("file %s is too large", fileName) + } + log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName) + return id, nil +} + +// Commit file stored in staging location by moving it to permanent location +func (f *FileSystem) Commit(id string) error { + log.Printf("[DEBUG] commit image %s", id) + stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id) + + if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil { + return errors.Wrap(err, "can't make image directory") + } + + err := os.Rename(stagingImage, permImage) + return errors.Wrapf(err, "failed to commit image %s", id) +} + +// Load image from FS. Uses id to get partition subdirectory. +// returns ReadCloser and caller should call close after processing completed. +func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) { + + // get image file by id. first try permanent location and if not found - staging + img := func(id string) (file string, st os.FileInfo, err error) { + file = f.location(f.Location, id) + st, err = os.Stat(file) + if err != nil { + file = f.location(f.Staging, id) + st, err = os.Stat(file) + } + return file, st, errors.Wrapf(err, "can't get image stats for %s", id) + } + + imgFile, st, err := img(id) + if err != nil { + return nil, 0, errors.Wrapf(err, "can't get image file for %s", id) + } + + fh, err := os.Open(imgFile) + if err != nil { + return nil, 0, errors.Wrapf(err, "can't load image %s", id) + } + return fh, st.Size(), nil +} + +// Cleanup runs scan of staging and removes old files based on ttl +func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error { + err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + age := time.Since(info.ModTime()) + if age > ttl { + log.Printf("[INFO] remove staging image %s, age %v", path, age) + return os.Remove(path) + } + return nil + }) + return errors.Wrap(err, "failed to cleanup images") +} + +// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories +// and avoid too many files in a single place. +// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png. +// Number of partitions defined by FileSystem.Partitions +func (f *FileSystem) location(base string, id string) string { + + partition := func(id string) string { + f.crc.Do(func() { + f.crc.Table = crc64.MakeTable(crc64.ECMA) + p := int(math.Round(math.Log10(float64(f.Partitions)))) + f.crc.mask = "%0" + strconv.Itoa(p) + "d" + f.crc.divider = uint64(math.Pow(10, float64(p))) + }) + checksum64 := crc64.Checksum([]byte(id), f.crc.Table) + partition := checksum64 % f.crc.divider + return fmt.Sprintf(f.crc.mask, partition) + } + + user, file := "unknown", id // default if no user in id + if elems := strings.Split(id, "/"); len(elems) == 2 { + user, file = elems[0], elems[1] // user in id + } + + if f.Partitions == 0 { + return path.Join(base, user, file) // avoid partition directory if 0 Partitions + } + + return path.Join(base, user, partition(id), file) +} diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go new file mode 100644 index 00000000..39c2abad --- /dev/null +++ b/backend/app/store/image/fs_store_test.go @@ -0,0 +1,213 @@ +package image + +import ( + "context" + "io/ioutil" + "math/rand" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFsStore_Save(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + + id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah")) + assert.NoError(t, err) + assert.Contains(t, id, "user1/") + assert.Contains(t, id, ".png") + t.Log(id) + + img := svc.location(svc.Staging, id) + t.Log(img) + data, err := ioutil.ReadFile(img) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) +} + +func TestFsStore_SaveAndCommit(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + + id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah")) + require.NoError(t, err) + err = svc.Commit(id) + require.NoError(t, err) + + imgStaging := svc.location(svc.Staging, id) + _, err = os.Stat(imgStaging) + assert.NotNil(t, err, "no file on staging anymore") + + img := svc.location(svc.Location, id) + t.Log(img) + data, err := ioutil.ReadFile(img) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) +} + +func TestFsStore_SaveTooLarge(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + svc.MaxSize = 5 + _, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah")) + assert.Error(t, err) + assert.Contains(t, err.Error(), "is too large") +} + +func TestFsStore_LoadAfterSave(t *testing.T) { + + svc, teardown := prepareImageTest(t) + defer teardown() + + id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah")) + assert.NoError(t, err) + t.Log(id) + + r, sz, err := svc.Load(id) + assert.NoError(t, err) + defer func() { assert.NoError(t, r.Close()) }() + data, err := ioutil.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) + assert.Equal(t, int64(9), sz) + _, _, err = svc.Load("abcd") + assert.NotNil(t, err) +} + +func TestFsStore_LoadAfterCommit(t *testing.T) { + + svc, teardown := prepareImageTest(t) + defer teardown() + + id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah")) + assert.NoError(t, err) + t.Log(id) + err = svc.Commit(id) + require.NoError(t, err) + + r, sz, err := svc.Load(id) + assert.NoError(t, err) + defer func() { assert.NoError(t, r.Close()) }() + data, err := ioutil.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, "blah blah", string(data)) + assert.Equal(t, int64(9), sz) + _, _, err = svc.Load("abcd") + assert.NotNil(t, err) +} + +func TestFsStore_location(t *testing.T) { + tbl := []struct { + partitions int + id, res string + }{ + {10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"}, + {10, "u2/abcdefe", "/tmp/u2/0/abcdefe"}, + {10, "u3/12345", "/tmp/u3/4/12345"}, + {100, "12345", "/tmp/unknown/69/12345"}, + {100, "xyzz", "/tmp/unknown/58/xyzz"}, + {100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"}, + {5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"}, + {5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"}, + {0, "12345", "/tmp/unknown/12345"}, + {0, "user/12345", "/tmp/user/12345"}, + } + for n, tt := range tbl { + t.Run(strconv.Itoa(n), func(t *testing.T) { + svc := FileSystem{Location: "/tmp", Partitions: tt.partitions} + assert.Equal(t, tt.res, svc.location("/tmp", tt.id)) + }) + } + + // generate random names and make sure partition never runs out of allowed + letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + randomID := func(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return "user1" + "/" + string(b) + } + + svc := FileSystem{Location: "/tmp", Partitions: 10} + for i := 0; i < 1000; i++ { + v := randomID(rand.Intn(64)) + location := svc.location("/tmp", v) + elems := strings.Split(location, "/") + p, err := strconv.Atoi(elems[3]) + require.NoError(t, err, location) + assert.True(t, p >= 0 && p < 10) + } +} + +func TestFsStore_Cleanup(t *testing.T) { + svc, teardown := prepareImageTest(t) + defer teardown() + + save := func(file string, user string, content string) (path string) { + id, err := svc.Save(file, user, strings.NewReader(content)) + require.NoError(t, err) + img := svc.location(svc.Staging, id) + data, err := ioutil.ReadFile(img) + require.NoError(t, err) + require.Equal(t, content, string(data)) + return img + } + + // save 3 images to staging + img1 := save("blah_ff1.png", "user1", "blah blah1") + time.Sleep(100 * time.Millisecond) + img2 := save("blah_ff2.png", "user1", "blah blah2") + time.Sleep(100 * time.Millisecond) + img3 := save("blah_ff3.png", "user2", "blah blah3") + + time.Sleep(100 * time.Millisecond) // make first image expired + err := svc.Cleanup(context.Background(), time.Millisecond*300) + assert.NoError(t, err) + + _, err = os.Stat(img1) + assert.NotNil(t, err, "no file on staging anymore") + _, err = os.Stat(img2) + assert.NoError(t, err, "file on staging") + _, err = os.Stat(img3) + assert.NoError(t, err, "file on staging") + + time.Sleep(200 * time.Millisecond) // make all images expired + err = svc.Cleanup(context.Background(), time.Millisecond*300) + assert.NoError(t, err) + + _, err = os.Stat(img2) + assert.NotNil(t, err, "no file on staging anymore") + _, err = os.Stat(img3) + assert.NotNil(t, err, "no file on staging anymore") +} + +func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) { + loc, err := ioutil.TempDir("", "test_image_r42") + require.NoError(t, err, "failed to make temp dir") + + staging, err := ioutil.TempDir("", "test_image_r42.staging") + require.NoError(t, err, "failed to make temp staging dir") + + svc = &FileSystem{ + Location: loc, + Staging: staging, + Partitions: 100, + MaxSize: 50, + } + + teardown = func() { + defer func() { + assert.NoError(t, os.RemoveAll(loc)) + assert.NoError(t, os.RemoveAll(staging)) + }() + } + + return svc, teardown +} diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go new file mode 100644 index 00000000..69230375 --- /dev/null +++ b/backend/app/store/image/image.go @@ -0,0 +1,127 @@ +// Package image handles storing, resizing and retrieval of images +// Provides Store with Save and Load and one implementation on top of local file system. +// Service object encloses Store and add common methods, this is the one consumer should use +package image + +//go:generate sh -c "mockgen -source=image.go -package=image > image_mock.go" + +import ( + "context" + "io" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/PuerkitoBio/goquery" + log "github.com/go-pkgz/lgr" + "github.com/pkg/errors" +) + +// Store defines interface for saving and loading pictures. +// Declares two-stage save with commit +type Store interface { + Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image + Commit(id string) error // move image from staging to permanent + Load(id string) (io.ReadCloser, int64, error) // load image by ID. Caller has to close the reader. + Cleanup(ctx context.Context, ttl time.Duration) error // run removal loop for old images on staging +} + +// Service extends Store with common functions needed for any store implementation +type Service struct { + Store + TTL time.Duration // for how long file allowed on staging + ImageAPI string // image api matching path + + wg sync.WaitGroup + submitCh chan submitReq + once sync.Once + term int32 +} + +const submitQueueSize = 5000 + +type submitReq struct { + idsFn func() (ids []string) + TS time.Time +} + +// Submit multiple ids via function for delayed commit +func (s *Service) Submit(idsFn func() []string) { + if idsFn == nil || s == nil { + return + } + + s.once.Do(func() { + log.Printf("[DEBUG] image submitter activated") + s.submitCh = make(chan submitReq, submitQueueSize) + s.wg.Add(1) + go func() { + defer s.wg.Done() + for req := range s.submitCh { + // wait for TTL expiration with emergency pass on term + for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL { + time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close) + } + for _, id := range req.idsFn() { + if err := s.Commit(id); err != nil { + log.Printf("[WARN] failed to commit image %s", id) + } + } + } + log.Printf("[INFO] image submitter terminated") + }() + }) + + s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()} +} + +// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png +func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) { + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML)) + if err != nil { + return nil, errors.Wrap(err, "can't create document") + } + result := []string{} + doc.Find("img").Each(func(i int, sl *goquery.Selection) { + if im, ok := sl.Attr("src"); ok { + if strings.Contains(im, s.ImageAPI) { + elems := strings.Split(im, "/") + if len(elems) >= 2 { + id := elems[len(elems)-2] + "/" + elems[len(elems)-1] + result = append(result, id) + } + } + } + }) + + return result, nil +} + +// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer +func (s *Service) Cleanup(ctx context.Context) { + log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL) + + for { + select { + case <-ctx.Done(): + log.Printf("[INFO] cleanup terminated, %v", ctx.Err()) + return + case <-time.After(s.TTL / 2): + if err := s.Store.Cleanup(ctx, s.TTL); err != nil { + log.Printf("[WARN] failed to cleanup, %v", err) + } + } + } +} + +// Close flushes all in-progress submits and enforces waiting commits +func (s *Service) Close() { + log.Printf("[INFO] close image service ") + atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh + if s.submitCh != nil { + close(s.submitCh) + } + s.wg.Wait() +} diff --git a/backend/app/store/image/image_mock.go b/backend/app/store/image/image_mock.go new file mode 100644 index 00000000..0049e558 --- /dev/null +++ b/backend/app/store/image/image_mock.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: image.go + +// Package image is a generated GoMock package. +package image + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + io "io" + reflect "reflect" + time "time" +) + +// MockStore is a mock of Store interface +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder +} + +// MockStoreMockRecorder is the mock recorder for MockStore +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Save mocks base method +func (m *MockStore) Save(fileName, userID string, r io.Reader) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", fileName, userID, r) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save +func (mr *MockStoreMockRecorder) Save(fileName, userID, r interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockStore)(nil).Save), fileName, userID, r) +} + +// Commit mocks base method +func (m *MockStore) Commit(id string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Commit", id) + ret0, _ := ret[0].(error) + return ret0 +} + +// Commit indicates an expected call of Commit +func (mr *MockStoreMockRecorder) Commit(id interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockStore)(nil).Commit), id) +} + +// Load mocks base method +func (m *MockStore) Load(id string) (io.ReadCloser, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", id) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Load indicates an expected call of Load +func (mr *MockStoreMockRecorder) Load(id interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockStore)(nil).Load), id) +} + +// Cleanup mocks base method +func (m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Cleanup", ctx, ttl) + ret0, _ := ret[0].(error) + return ret0 +} + +// Cleanup indicates an expected call of Cleanup +func (mr *MockStoreMockRecorder) Cleanup(ctx, ttl interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockStore)(nil).Cleanup), ctx, ttl) +} diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go new file mode 100644 index 00000000..f26eec8a --- /dev/null +++ b/backend/app/store/image/image_test.go @@ -0,0 +1,77 @@ +package image + +import ( + "context" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestService_ExtractPictures(t *testing.T) { + svc := Service{ImageAPI: "/blah/"} + html := `blah foo + xyz

123

` + ids, err := svc.ExtractPictures(html) + require.NoError(t, err) + assert.Equal(t, 2, len(ids), "two images") + assert.Equal(t, "user1/pic1.png", ids[0]) + assert.Equal(t, "user2/pic3.png", ids[1]) +} + +func TestService_Cleanup(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := NewMockStore(ctrl) + store.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Times(10) + + svc := Service{Store: store, TTL: 100 * time.Millisecond} + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549) + defer cancel() + svc.Cleanup(ctx) +} + +func TestService_Submit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + store := NewMockStore(ctrl) + + store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed + svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(nil) + time.Sleep(time.Millisecond * 500) +} + +func TestService_Close(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + store := NewMockStore(ctrl) + + store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed + svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500} + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + svc.Submit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(nil) + svc.Close() +} + +func TestService_SubmitDelay(t *testing.T) { + ctrl := gomock.NewController(t) + defer func() { + ctrl.Finish() + }() + + store := NewMockStore(ctrl) + + store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed + svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100} + svc.Submit(func() []string { return []string{"id1", "id2", "id3"} }) + time.Sleep(150 * time.Millisecond) // let first batch to pass TTL + svc.Submit(func() []string { return []string{"id4", "id5"} }) + svc.Submit(nil) +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 4878dae9..fe30a28c 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -12,6 +12,7 @@ import ( multierror "github.com/hashicorp/go-multierror" cache "github.com/patrickmn/go-cache" "github.com/pkg/errors" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" @@ -28,6 +29,7 @@ type DataStore struct { PositiveScore bool TitleExtractor *TitleExtractor RestrictedWordsMatcher *RestrictedWordsMatcher + ImageService *image.Service // granular locks scopedLocks struct { @@ -90,9 +92,32 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) comment.PostTitle = title }() + s.submitImages(comment) return s.Interface.Create(comment) } +// submitImages initiated delayed commit of all images from the comment uploaded to remark42 +func (s *DataStore) submitImages(comment store.Comment) { + + s.ImageService.Submit(func() []string { + c := comment + cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment + if err != nil { + log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err) + return nil + } + imgIds, err := s.ImageService.ExtractPictures(cc.Text) + if err != nil { + log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err) + return nil + } + if len(imgIds) > 0 { + log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds) + } + return imgIds + }) +} + // prepareNewComment sets new comment fields, hashing and sanitizing data func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) { // fill ID and time if empty @@ -129,8 +154,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) // Vote for comment by id and locator func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) { - cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope - cLock.Lock() // prevents race on voting + cLock := s.getScopedLocks(locator.URL) // get lock for URL scope + cLock.Lock() // prevents race on voting defer cLock.Unlock() comment, err = s.Get(locator, commentID) @@ -455,8 +480,8 @@ func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) { return ups, downs } -// getsScopedLocks pull lock from the map if found or create a new one -func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) { +// getScopedLocks pull lock from the map if found or create a new one +func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) { s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} }) s.scopedLocks.Lock() diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 2612e945..0918c210 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -13,9 +13,12 @@ import ( "time" bolt "github.com/coreos/bbolt" + "github.com/go-pkgz/lgr" + "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" @@ -686,6 +689,35 @@ func TestService_Find(t *testing.T) { assert.InDelta(t, 0, res[1].Controversy, 0.01) } +func TestService_submitImages(t *testing.T) { + defer os.Remove(testDb) + lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockStore := image.NewMockStore(ctrl) + imgSvc := &image.Service{Store: mockStore, TTL: time.Millisecond * 50} + + mockStore.EXPECT().Commit(gomock.Any()).Times(2) + + // two comments for https://radio-t.com + b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond, + AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc} + + c := store.Comment{ + ID: "id-22", + Text: `some text xx `, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages + assert.NoError(t, err) + + b.submitImages(c) + time.Sleep(250 * time.Millisecond) +} + // makes new boltdb, put two records func prepStoreEngine(t *testing.T) engine.Interface { os.Remove(testDb) diff --git a/backend/go.mod b/backend/go.mod index 6dc84b44..071bec33 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -20,20 +20,20 @@ require ( github.com/go-pkgz/repeater v1.1.1 github.com/go-pkgz/rest v1.4.0 github.com/go-pkgz/syncs v1.1.0 + github.com/golang/mock v1.2.0 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c github.com/gorilla/feeds v1.1.0 - github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 - github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.8.1 github.com/rakyll/statik v0.1.3 - github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/stretchr/testify v1.3.0 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 - golang.org/x/time v0.0.0-20170927054726-6dc17368e09b // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 ) diff --git a/backend/go.sum b/backend/go.sum index 7a21bfa4..7327b7e7 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -45,20 +45,20 @@ github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk= github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk= github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY= github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk= github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8= github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw= github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -79,9 +79,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU= github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -101,8 +102,8 @@ golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYb golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU= -golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/backend/vendor/github.com/golang/mock/AUTHORS b/backend/vendor/github.com/golang/mock/AUTHORS new file mode 100644 index 00000000..660b8ccc --- /dev/null +++ b/backend/vendor/github.com/golang/mock/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of GoMock authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Alex Reece +Google Inc. diff --git a/backend/vendor/github.com/golang/mock/CONTRIBUTORS b/backend/vendor/github.com/golang/mock/CONTRIBUTORS new file mode 100644 index 00000000..def849ca --- /dev/null +++ b/backend/vendor/github.com/golang/mock/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute (and typically +# have contributed) code to the gomock repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name +# +# An entry with two email addresses specifies that the +# first address should be used in the submit logs and +# that the second address should be recognized as the +# same person when interacting with Rietveld. + +# Please keep the list sorted. + +Aaron Jacobs +Alex Reece +David Symonds +Ryan Barrett diff --git a/backend/vendor/github.com/golang/mock/LICENSE b/backend/vendor/github.com/golang/mock/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/backend/vendor/github.com/golang/mock/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/backend/vendor/github.com/golang/mock/gomock/call.go b/backend/vendor/github.com/golang/mock/gomock/call.go new file mode 100644 index 00000000..3d54d9f5 --- /dev/null +++ b/backend/vendor/github.com/golang/mock/gomock/call.go @@ -0,0 +1,420 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +// Call represents an expected call to a mock. +type Call struct { + t TestHelper // for triggering test failures on invalid call setup + + receiver interface{} // the receiver of the method call + method string // the name of the method + methodType reflect.Type // the type of the method + args []Matcher // the args + origin string // file and line number of call setup + + preReqs []*Call // prerequisite calls + + // Expectations + minCalls, maxCalls int + + numCalls int // actual number made + + // actions are called when this Call is called. Each action gets the args and + // can set the return values by returning a non-nil slice. Actions run in the + // order they are created. + actions []func([]interface{}) []interface{} +} + +// newCall creates a *Call. It requires the method type in order to support +// unexported methods. +func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + t.Helper() + + // TODO: check arity, types. + margs := make([]Matcher, len(args)) + for i, arg := range args { + if m, ok := arg.(Matcher); ok { + margs[i] = m + } else if arg == nil { + // Handle nil specially so that passing a nil interface value + // will match the typed nils of concrete args. + margs[i] = Nil() + } else { + margs[i] = Eq(arg) + } + } + + origin := callerInfo(3) + actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} { + // Synthesize the zero value for each of the return args' types. + rets := make([]interface{}, methodType.NumOut()) + for i := 0; i < methodType.NumOut(); i++ { + rets[i] = reflect.Zero(methodType.Out(i)).Interface() + } + return rets + }} + return &Call{t: t, receiver: receiver, method: method, methodType: methodType, + args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions} +} + +// AnyTimes allows the expectation to be called 0 or more times +func (c *Call) AnyTimes() *Call { + c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity + return c +} + +// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also +// sets the maximum number of calls to infinity. +func (c *Call) MinTimes(n int) *Call { + c.minCalls = n + if c.maxCalls == 1 { + c.maxCalls = 1e8 + } + return c +} + +// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also +// sets the minimum number of calls to 0. +func (c *Call) MaxTimes(n int) *Call { + c.maxCalls = n + if c.minCalls == 1 { + c.minCalls = 0 + } + return c +} + +// DoAndReturn declares the action to run when the call is matched. +// The return values from this function are returned by the mocked function. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) DoAndReturn(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + vargs := make([]reflect.Value, len(args)) + ft := v.Type() + for i := 0; i < len(args); i++ { + if args[i] != nil { + vargs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vargs[i] = reflect.Zero(ft.In(i)) + } + } + vrets := v.Call(vargs) + rets := make([]interface{}, len(vrets)) + for i, ret := range vrets { + rets[i] = ret.Interface() + } + return rets + }) + return c +} + +// Do declares the action to run when the call is matched. The function's +// return values are ignored to retain backward compatibility. To use the +// return values call DoAndReturn. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) Do(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + vargs := make([]reflect.Value, len(args)) + ft := v.Type() + for i := 0; i < len(args); i++ { + if args[i] != nil { + vargs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vargs[i] = reflect.Zero(ft.In(i)) + } + } + v.Call(vargs) + return nil + }) + return c +} + +// Return declares the values to be returned by the mocked function call. +func (c *Call) Return(rets ...interface{}) *Call { + c.t.Helper() + + mt := c.methodType + if len(rets) != mt.NumOut() { + c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]", + c.receiver, c.method, len(rets), mt.NumOut(), c.origin) + } + for i, ret := range rets { + if got, want := reflect.TypeOf(ret), mt.Out(i); got == want { + // Identical types; nothing to do. + } else if got == nil { + // Nil needs special handling. + switch want.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + // ok + default: + c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]", + i, c.receiver, c.method, want, c.origin) + } + } else if got.AssignableTo(want) { + // Assignable type relation. Make the assignment now so that the generated code + // can return the values with a type assertion. + v := reflect.New(want).Elem() + v.Set(reflect.ValueOf(ret)) + rets[i] = v.Interface() + } else { + c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]", + i, c.receiver, c.method, got, want, c.origin) + } + } + + c.addAction(func([]interface{}) []interface{} { + return rets + }) + + return c +} + +// Times declares the exact number of times a function call is expected to be executed. +func (c *Call) Times(n int) *Call { + c.minCalls, c.maxCalls = n, n + return c +} + +// SetArg declares an action that will set the nth argument's value, +// indirected through a pointer. Or, in the case of a slice, SetArg +// will copy value's elements into the nth argument. +func (c *Call) SetArg(n int, value interface{}) *Call { + c.t.Helper() + + mt := c.methodType + // TODO: This will break on variadic methods. + // We will need to check those at invocation time. + if n < 0 || n >= mt.NumIn() { + c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]", + n, mt.NumIn(), c.origin) + } + // Permit setting argument through an interface. + // In the interface case, we don't (nay, can't) check the type here. + at := mt.In(n) + switch at.Kind() { + case reflect.Ptr: + dt := at.Elem() + if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) { + c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]", + n, vt, dt, c.origin) + } + case reflect.Interface: + // nothing to do + case reflect.Slice: + // nothing to do + default: + c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]", + n, at, c.origin) + } + + c.addAction(func(args []interface{}) []interface{} { + v := reflect.ValueOf(value) + switch reflect.TypeOf(args[n]).Kind() { + case reflect.Slice: + setSlice(args[n], v) + default: + reflect.ValueOf(args[n]).Elem().Set(v) + } + return nil + }) + return c +} + +// isPreReq returns true if other is a direct or indirect prerequisite to c. +func (c *Call) isPreReq(other *Call) bool { + for _, preReq := range c.preReqs { + if other == preReq || preReq.isPreReq(other) { + return true + } + } + return false +} + +// After declares that the call may only match after preReq has been exhausted. +func (c *Call) After(preReq *Call) *Call { + c.t.Helper() + + if c == preReq { + c.t.Fatalf("A call isn't allowed to be its own prerequisite") + } + if preReq.isPreReq(c) { + c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq) + } + + c.preReqs = append(c.preReqs, preReq) + return c +} + +// Returns true if the minimum number of calls have been made. +func (c *Call) satisfied() bool { + return c.numCalls >= c.minCalls +} + +// Returns true iff the maximum number of calls have been made. +func (c *Call) exhausted() bool { + return c.numCalls >= c.maxCalls +} + +func (c *Call) String() string { + args := make([]string, len(c.args)) + for i, arg := range c.args { + args[i] = arg.String() + } + arguments := strings.Join(args, ", ") + return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin) +} + +// Tests if the given call matches the expected call. +// If yes, returns nil. If no, returns error with message explaining why it does not match. +func (c *Call) matches(args []interface{}) error { + if !c.methodType.IsVariadic() { + if len(args) != len(c.args) { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + + for i, m := range c.args { + if !m.Matches(args[i]) { + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i], m) + } + } + } else { + if len(c.args) < c.methodType.NumIn()-1 { + return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d", + c.origin, len(c.args), c.methodType.NumIn()-1) + } + if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + if len(args) < len(c.args)-1 { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d", + c.origin, len(args), len(c.args)-1) + } + + for i, m := range c.args { + if i < c.methodType.NumIn()-1 { + // Non-variadic args + if !m.Matches(args[i]) { + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i], m) + } + continue + } + // The last arg has a possibility of a variadic argument, so let it branch + + // sample: Foo(a int, b int, c ...int) + if i < len(c.args) && i < len(args) { + if m.Matches(args[i]) { + // Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC) + // Got Foo(a, b) want Foo(matcherA, matcherB) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD) + continue + } + } + + // The number of actual args don't match the number of matchers, + // or the last matcher is a slice and the last arg is not. + // If this function still matches it is because the last matcher + // matches all the remaining arguments or the lack of any. + // Convert the remaining arguments, if any, into a slice of the + // expected type. + vargsType := c.methodType.In(c.methodType.NumIn() - 1) + vargs := reflect.MakeSlice(vargsType, 0, len(args)-i) + for _, arg := range args[i:] { + vargs = reflect.Append(vargs, reflect.ValueOf(arg)) + } + if m.Matches(vargs.Interface()) { + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher) + break + } + // Wrong number of matchers or not match. Fail. + // Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB) + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i:], c.args[i]) + + } + } + + // Check that all prerequisite calls have been satisfied. + for _, preReqCall := range c.preReqs { + if !preReqCall.satisfied() { + return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v", + c.origin, preReqCall, c) + } + } + + // Check that the call is not exhausted. + if c.exhausted() { + return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin) + } + + return nil +} + +// dropPrereqs tells the expected Call to not re-check prerequisite calls any +// longer, and to return its current set. +func (c *Call) dropPrereqs() (preReqs []*Call) { + preReqs = c.preReqs + c.preReqs = nil + return +} + +func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} { + c.numCalls++ + return c.actions +} + +// InOrder declares that the given calls should occur in order. +func InOrder(calls ...*Call) { + for i := 1; i < len(calls); i++ { + calls[i].After(calls[i-1]) + } +} + +func setSlice(arg interface{}, v reflect.Value) { + va := reflect.ValueOf(arg) + for i := 0; i < v.Len(); i++ { + va.Index(i).Set(v.Index(i)) + } +} + +func (c *Call) addAction(action func([]interface{}) []interface{}) { + c.actions = append(c.actions, action) +} diff --git a/backend/vendor/github.com/golang/mock/gomock/callset.go b/backend/vendor/github.com/golang/mock/gomock/callset.go new file mode 100644 index 00000000..c44a8a58 --- /dev/null +++ b/backend/vendor/github.com/golang/mock/gomock/callset.go @@ -0,0 +1,108 @@ +// Copyright 2011 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "bytes" + "fmt" +) + +// callSet represents a set of expected calls, indexed by receiver and method +// name. +type callSet struct { + // Calls that are still expected. + expected map[callSetKey][]*Call + // Calls that have been exhausted. + exhausted map[callSetKey][]*Call +} + +// callSetKey is the key in the maps in callSet +type callSetKey struct { + receiver interface{} + fname string +} + +func newCallSet() *callSet { + return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)} +} + +// Add adds a new expected call. +func (cs callSet) Add(call *Call) { + key := callSetKey{call.receiver, call.method} + m := cs.expected + if call.exhausted() { + m = cs.exhausted + } + m[key] = append(m[key], call) +} + +// Remove removes an expected call. +func (cs callSet) Remove(call *Call) { + key := callSetKey{call.receiver, call.method} + calls := cs.expected[key] + for i, c := range calls { + if c == call { + // maintain order for remaining calls + cs.expected[key] = append(calls[:i], calls[i+1:]...) + cs.exhausted[key] = append(cs.exhausted[key], call) + break + } + } +} + +// FindMatch searches for a matching call. Returns error with explanation message if no call matched. +func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) { + key := callSetKey{receiver, method} + + // Search through the expected calls. + expected := cs.expected[key] + var callsErrors bytes.Buffer + for _, call := range expected { + err := call.matches(args) + if err != nil { + fmt.Fprintf(&callsErrors, "\n%v", err) + } else { + return call, nil + } + } + + // If we haven't found a match then search through the exhausted calls so we + // get useful error messages. + exhausted := cs.exhausted[key] + for _, call := range exhausted { + if err := call.matches(args); err != nil { + fmt.Fprintf(&callsErrors, "\n%v", err) + } + } + + if len(expected)+len(exhausted) == 0 { + fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method) + } + + return nil, fmt.Errorf(callsErrors.String()) +} + +// Failures returns the calls that are not satisfied. +func (cs callSet) Failures() []*Call { + failures := make([]*Call, 0, len(cs.expected)) + for _, calls := range cs.expected { + for _, call := range calls { + if !call.satisfied() { + failures = append(failures, call) + } + } + } + return failures +} diff --git a/backend/vendor/github.com/golang/mock/gomock/controller.go b/backend/vendor/github.com/golang/mock/gomock/controller.go new file mode 100644 index 00000000..6fde25f5 --- /dev/null +++ b/backend/vendor/github.com/golang/mock/gomock/controller.go @@ -0,0 +1,235 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GoMock - a mock framework for Go. +// +// Standard usage: +// (1) Define an interface that you wish to mock. +// type MyInterface interface { +// SomeMethod(x int64, y string) +// } +// (2) Use mockgen to generate a mock from the interface. +// (3) Use the mock in a test: +// func TestMyThing(t *testing.T) { +// mockCtrl := gomock.NewController(t) +// defer mockCtrl.Finish() +// +// mockObj := something.NewMockMyInterface(mockCtrl) +// mockObj.EXPECT().SomeMethod(4, "blah") +// // pass mockObj to a real object and play with it. +// } +// +// By default, expected calls are not enforced to run in any particular order. +// Call order dependency can be enforced by use of InOrder and/or Call.After. +// Call.After can create more varied call order dependencies, but InOrder is +// often more convenient. +// +// The following examples create equivalent call order dependencies. +// +// Example of using Call.After to chain expected call order: +// +// firstCall := mockObj.EXPECT().SomeMethod(1, "first") +// secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall) +// mockObj.EXPECT().SomeMethod(3, "third").After(secondCall) +// +// Example of using InOrder to declare expected call order: +// +// gomock.InOrder( +// mockObj.EXPECT().SomeMethod(1, "first"), +// mockObj.EXPECT().SomeMethod(2, "second"), +// mockObj.EXPECT().SomeMethod(3, "third"), +// ) +// +// TODO: +// - Handle different argument/return types (e.g. ..., chan, map, interface). +package gomock + +import ( + "context" + "fmt" + "reflect" + "runtime" + "sync" +) + +// A TestReporter is something that can be used to report test failures. +// It is satisfied by the standard library's *testing.T. +type TestReporter interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) +} + +// TestHelper is a TestReporter that has the Helper method. It is satisfied +// by the standard library's *testing.T. +type TestHelper interface { + TestReporter + Helper() +} + +// A Controller represents the top-level control of a mock ecosystem. +// It defines the scope and lifetime of mock objects, as well as their expectations. +// It is safe to call Controller's methods from multiple goroutines. +type Controller struct { + // T should only be called within a generated mock. It is not intended to + // be used in user code and may be changed in future versions. T is the + // TestReporter passed in when creating the Controller via NewController. + // If the TestReporter does not implment a TestHelper it will be wrapped + // with a nopTestHelper. + T TestHelper + mu sync.Mutex + expectedCalls *callSet + finished bool +} + +func NewController(t TestReporter) *Controller { + h, ok := t.(TestHelper) + if !ok { + h = nopTestHelper{t} + } + + return &Controller{ + T: h, + expectedCalls: newCallSet(), + } +} + +type cancelReporter struct { + TestHelper + cancel func() +} + +func (r *cancelReporter) Errorf(format string, args ...interface{}) { + r.TestHelper.Errorf(format, args...) +} +func (r *cancelReporter) Fatalf(format string, args ...interface{}) { + defer r.cancel() + r.TestHelper.Fatalf(format, args...) +} + +// WithContext returns a new Controller and a Context, which is cancelled on any +// fatal failure. +func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) { + h, ok := t.(TestHelper) + if !ok { + h = nopTestHelper{t} + } + + ctx, cancel := context.WithCancel(ctx) + return NewController(&cancelReporter{h, cancel}), ctx +} + +type nopTestHelper struct { + TestReporter +} + +func (h nopTestHelper) Helper() {} + +func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call { + ctrl.T.Helper() + + recv := reflect.ValueOf(receiver) + for i := 0; i < recv.Type().NumMethod(); i++ { + if recv.Type().Method(i).Name == method { + return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...) + } + } + ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver) + panic("unreachable") +} + +func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + ctrl.T.Helper() + + call := newCall(ctrl.T, receiver, method, methodType, args...) + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + ctrl.expectedCalls.Add(call) + + return call +} + +func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} { + ctrl.T.Helper() + + // Nest this code so we can use defer to make sure the lock is released. + actions := func() []func([]interface{}) []interface{} { + ctrl.T.Helper() + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args) + if err != nil { + origin := callerInfo(2) + ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err) + } + + // Two things happen here: + // * the matching call no longer needs to check prerequite calls, + // * and the prerequite calls are no longer expected, so remove them. + preReqCalls := expected.dropPrereqs() + for _, preReqCall := range preReqCalls { + ctrl.expectedCalls.Remove(preReqCall) + } + + actions := expected.call(args) + if expected.exhausted() { + ctrl.expectedCalls.Remove(expected) + } + return actions + }() + + var rets []interface{} + for _, action := range actions { + if r := action(args); r != nil { + rets = r + } + } + + return rets +} + +func (ctrl *Controller) Finish() { + ctrl.T.Helper() + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + if ctrl.finished { + ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.") + } + ctrl.finished = true + + // If we're currently panicking, probably because this is a deferred call, + // pass through the panic. + if err := recover(); err != nil { + panic(err) + } + + // Check that all remaining expected calls are satisfied. + failures := ctrl.expectedCalls.Failures() + for _, call := range failures { + ctrl.T.Errorf("missing call(s) to %v", call) + } + if len(failures) != 0 { + ctrl.T.Fatalf("aborting test due to missing call(s)") + } +} + +func callerInfo(skip int) string { + if _, file, line, ok := runtime.Caller(skip + 1); ok { + return fmt.Sprintf("%s:%d", file, line) + } + return "unknown file" +} diff --git a/backend/vendor/github.com/golang/mock/gomock/matchers.go b/backend/vendor/github.com/golang/mock/gomock/matchers.go new file mode 100644 index 00000000..189796f8 --- /dev/null +++ b/backend/vendor/github.com/golang/mock/gomock/matchers.go @@ -0,0 +1,122 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" +) + +// A Matcher is a representation of a class of values. +// It is used to represent the valid or expected arguments to a mocked method. +type Matcher interface { + // Matches returns whether x is a match. + Matches(x interface{}) bool + + // String describes what the matcher matches. + String() string +} + +type anyMatcher struct{} + +func (anyMatcher) Matches(x interface{}) bool { + return true +} + +func (anyMatcher) String() string { + return "is anything" +} + +type eqMatcher struct { + x interface{} +} + +func (e eqMatcher) Matches(x interface{}) bool { + return reflect.DeepEqual(e.x, x) +} + +func (e eqMatcher) String() string { + return fmt.Sprintf("is equal to %v", e.x) +} + +type nilMatcher struct{} + +func (nilMatcher) Matches(x interface{}) bool { + if x == nil { + return true + } + + v := reflect.ValueOf(x) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice: + return v.IsNil() + } + + return false +} + +func (nilMatcher) String() string { + return "is nil" +} + +type notMatcher struct { + m Matcher +} + +func (n notMatcher) Matches(x interface{}) bool { + return !n.m.Matches(x) +} + +func (n notMatcher) String() string { + // TODO: Improve this if we add a NotString method to the Matcher interface. + return "not(" + n.m.String() + ")" +} + +type assignableToTypeOfMatcher struct { + targetType reflect.Type +} + +func (m assignableToTypeOfMatcher) Matches(x interface{}) bool { + return reflect.TypeOf(x).AssignableTo(m.targetType) +} + +func (m assignableToTypeOfMatcher) String() string { + return "is assignable to " + m.targetType.Name() +} + +// Constructors +func Any() Matcher { return anyMatcher{} } +func Eq(x interface{}) Matcher { return eqMatcher{x} } +func Nil() Matcher { return nilMatcher{} } +func Not(x interface{}) Matcher { + if m, ok := x.(Matcher); ok { + return notMatcher{m} + } + return notMatcher{Eq(x)} +} + +// AssignableToTypeOf is a Matcher that matches if the parameter to the mock +// function is assignable to the type of the parameter to this function. +// +// Example usage: +// +// dbMock.EXPECT(). +// Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})). +// Return(errors.New("DB error")) +// +func AssignableToTypeOf(x interface{}) Matcher { + return assignableToTypeOfMatcher{reflect.TypeOf(x)} +} diff --git a/backend/vendor/github.com/hashicorp/errwrap/README.md b/backend/vendor/github.com/hashicorp/errwrap/README.md index 1c95f597..444df08f 100644 --- a/backend/vendor/github.com/hashicorp/errwrap/README.md +++ b/backend/vendor/github.com/hashicorp/errwrap/README.md @@ -48,7 +48,7 @@ func main() { // We can use the Contains helpers to check if an error contains // another error. It is safe to do this with a nil error, or with // an error that doesn't even use the errwrap package. - if errwrap.Contains(err, ErrNotExist) { + if errwrap.Contains(err, "does not exist") { // Do something } if errwrap.ContainsType(err, new(os.PathError)) { diff --git a/backend/vendor/github.com/hashicorp/errwrap/go.mod b/backend/vendor/github.com/hashicorp/errwrap/go.mod new file mode 100644 index 00000000..c9b84022 --- /dev/null +++ b/backend/vendor/github.com/hashicorp/errwrap/go.mod @@ -0,0 +1 @@ +module github.com/hashicorp/errwrap diff --git a/backend/vendor/github.com/hashicorp/golang-lru/lru.go b/backend/vendor/github.com/hashicorp/golang-lru/lru.go index 1cbe04b7..c8d9b0a2 100644 --- a/backend/vendor/github.com/hashicorp/golang-lru/lru.go +++ b/backend/vendor/github.com/hashicorp/golang-lru/lru.go @@ -40,35 +40,31 @@ func (c *Cache) Purge() { // Add adds a value to the cache. Returns true if an eviction occurred. func (c *Cache) Add(key, value interface{}) (evicted bool) { c.lock.Lock() - evicted = c.lru.Add(key, value) - c.lock.Unlock() - return evicted + defer c.lock.Unlock() + return c.lru.Add(key, value) } // Get looks up a key's value from the cache. func (c *Cache) Get(key interface{}) (value interface{}, ok bool) { c.lock.Lock() - value, ok = c.lru.Get(key) - c.lock.Unlock() - return value, ok + defer c.lock.Unlock() + return c.lru.Get(key) } // Contains checks if a key is in the cache, without updating the // recent-ness or deleting it for being stale. func (c *Cache) Contains(key interface{}) bool { c.lock.RLock() - containKey := c.lru.Contains(key) - c.lock.RUnlock() - return containKey + defer c.lock.RUnlock() + return c.lru.Contains(key) } // Peek returns the key value (or undefined if not found) without updating // the "recently used"-ness of the key. func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) { c.lock.RLock() - value, ok = c.lru.Peek(key) - c.lock.RUnlock() - return value, ok + defer c.lock.RUnlock() + return c.lru.Peek(key) } // ContainsOrAdd checks if a key is in the cache without updating the @@ -102,15 +98,13 @@ func (c *Cache) RemoveOldest() { // Keys returns a slice of the keys in the cache, from oldest to newest. func (c *Cache) Keys() []interface{} { c.lock.RLock() - keys := c.lru.Keys() - c.lock.RUnlock() - return keys + defer c.lock.RUnlock() + return c.lru.Keys() } // Len returns the number of items in the cache. func (c *Cache) Len() int { c.lock.RLock() - length := c.lru.Len() - c.lock.RUnlock() - return length + defer c.lock.RUnlock() + return c.lru.Len() } diff --git a/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod b/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod new file mode 100644 index 00000000..1e255347 --- /dev/null +++ b/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod @@ -0,0 +1 @@ +module github.com/shurcooL/sanitized_anchor_name diff --git a/backend/vendor/golang.org/x/time/rate/rate.go b/backend/vendor/golang.org/x/time/rate/rate.go index eabcd114..ae93e247 100644 --- a/backend/vendor/golang.org/x/time/rate/rate.go +++ b/backend/vendor/golang.org/x/time/rate/rate.go @@ -6,6 +6,7 @@ package rate import ( + "context" "fmt" "math" "sync" @@ -212,19 +213,8 @@ func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation { return &r } -// contextContext is a temporary(?) copy of the context.Context type -// to support both Go 1.6 using golang.org/x/net/context and Go 1.7+ -// with the built-in context package. If people ever stop using Go 1.6 -// we can remove this. -type contextContext interface { - Deadline() (deadline time.Time, ok bool) - Done() <-chan struct{} - Err() error - Value(key interface{}) interface{} -} - // Wait is shorthand for WaitN(ctx, 1). -func (lim *Limiter) wait(ctx contextContext) (err error) { +func (lim *Limiter) Wait(ctx context.Context) (err error) { return lim.WaitN(ctx, 1) } @@ -232,7 +222,7 @@ func (lim *Limiter) wait(ctx contextContext) (err error) { // It returns an error if n exceeds the Limiter's burst size, the Context is // canceled, or the expected wait time exceeds the Context's Deadline. // The burst limit is ignored if the rate limit is Inf. -func (lim *Limiter) waitN(ctx contextContext, n int) (err error) { +func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { if n > lim.burst && lim.limit != Inf { return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, lim.burst) } @@ -253,8 +243,12 @@ func (lim *Limiter) waitN(ctx contextContext, n int) (err error) { if !r.ok { return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n) } - // Wait - t := time.NewTimer(r.DelayFrom(now)) + // Wait if necessary + delay := r.DelayFrom(now) + if delay == 0 { + return nil + } + t := time.NewTimer(delay) defer t.Stop() select { case <-t.C: diff --git a/backend/vendor/golang.org/x/time/rate/rate_go16.go b/backend/vendor/golang.org/x/time/rate/rate_go16.go deleted file mode 100644 index 6bab1850..00000000 --- a/backend/vendor/golang.org/x/time/rate/rate_go16.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !go1.7 - -package rate - -import "golang.org/x/net/context" - -// Wait is shorthand for WaitN(ctx, 1). -func (lim *Limiter) Wait(ctx context.Context) (err error) { - return lim.waitN(ctx, 1) -} - -// WaitN blocks until lim permits n events to happen. -// It returns an error if n exceeds the Limiter's burst size, the Context is -// canceled, or the expected wait time exceeds the Context's Deadline. -func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { - return lim.waitN(ctx, n) -} diff --git a/backend/vendor/golang.org/x/time/rate/rate_go17.go b/backend/vendor/golang.org/x/time/rate/rate_go17.go deleted file mode 100644 index f90d85f5..00000000 --- a/backend/vendor/golang.org/x/time/rate/rate_go17.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build go1.7 - -package rate - -import "context" - -// Wait is shorthand for WaitN(ctx, 1). -func (lim *Limiter) Wait(ctx context.Context) (err error) { - return lim.waitN(ctx, 1) -} - -// WaitN blocks until lim permits n events to happen. -// It returns an error if n exceeds the Limiter's burst size, the Context is -// canceled, or the expected wait time exceeds the Context's Deadline. -func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { - return lim.waitN(ctx, n) -} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 4863c4d0..60f43d75 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -52,17 +52,19 @@ github.com/go-pkgz/rest github.com/go-pkgz/rest/logger # github.com/go-pkgz/syncs v1.1.0 github.com/go-pkgz/syncs +# github.com/golang/mock v1.2.0 +github.com/golang/mock/gomock # github.com/golang/protobuf v1.2.0 github.com/golang/protobuf/proto # github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c github.com/google/uuid # github.com/gorilla/feeds v1.1.0 github.com/gorilla/feeds -# github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce +# github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/errwrap # github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 github.com/hashicorp/go-multierror -# github.com/hashicorp/golang-lru v0.5.1 +# github.com/hashicorp/golang-lru v0.5.0 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru # github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc @@ -79,7 +81,7 @@ github.com/pkg/errors github.com/pmezard/go-difflib/difflib # github.com/rakyll/statik v0.1.3 github.com/rakyll/statik/fs -# github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 +# github.com/shurcooL/sanitized_anchor_name v1.0.0 github.com/shurcooL/sanitized_anchor_name # github.com/stretchr/testify v1.3.0 github.com/stretchr/testify/assert @@ -106,7 +108,7 @@ golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb golang.org/x/sys/unix -# golang.org/x/time v0.0.0-20170927054726-6dc17368e09b +# golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/time/rate # google.golang.org/appengine v1.4.0 google.golang.org/appengine