Merge pull request #296 from umputun/image

Image storage

.gitignore

@@ -18,4 +18,4 @@ debug.test
 .mongo
 remark42
 /bin/
-/backend/var/
+/backend/var/

Dockerfile

@@ -29,7 +29,9 @@ WORKDIR /build/backend
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test -mod=vendor -covermode=count -coverprofile=/profile.cov ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
 RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
@@ -37,10 +39,10 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-    --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-    --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-    --enable=ineffassign --enable=varcheck ./... ; \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
 # submit coverage to coverals if COVERALLS_TOKEN in env
@@ -50,8 +52,7 @@ RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
     go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app

README.md

@@ -86,63 +86,68 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)        | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                  | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | `false`               | local oauth2 server, development mode only       |
-| auth.anon               | AUTH_ANON               | `false`               | enable anonymous login                           |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | `100`                 | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                  | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | `8443`                | port for https server                            |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | `2048`                | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
-| positive-score          | POSITIVE_SCORE          | `false`               | enable positive score only                       |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
-| positive-score          | POSITIVE_SCORE          | `false`               | restricts comment's score to be only positive    |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            | none (disabled)       | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                      |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, 'bolt`, or `mongo` |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -602,6 +607,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image 
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+  
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.

backend/app/cmd/avatar.go

@@ -31,6 +31,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }

backend/app/cmd/cleanup.go

@@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil

backend/app/cmd/server.go

@@ -15,7 +15,7 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	auth_cache "github.com/patrickmn/go-cache"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
@@ -32,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -43,6 +44,7 @@ type ServerCommand struct {
 	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -93,6 +95,20 @@ type StoreGroup struct {
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
 }
 
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+}
+
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
@@ -162,6 +178,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -182,6 +199,7 @@ func (s *ServerCommand) Execute(args []string) error {
 	app, err := s.newServerApp()
 	if err != nil {
 		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
 		log.Printf("[ERROR] remark terminated with error %+v", err)
@@ -214,6 +232,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+
 	dataService := &service.DataStore{
 		Interface:              storeEngine,
 		EditDuration:           s.EditDuration,
@@ -221,6 +244,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
 		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
@@ -276,15 +300,16 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     imageService,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+		da, errDevAuth := authenticator.DevAuth()
+		if errDevAuth != nil {
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -298,6 +323,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -323,12 +349,17 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -405,6 +436,25 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+			},
+			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
+		}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
@@ -585,17 +635,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*auth_cache.Cache
+	*authcache.Cache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
 }
 
+// Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 	return c.Cache.Get(key.(string))
 }
 
+// Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
 }

backend/app/cmd/server_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})

backend/app/migrator/disqus.go

@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue

backend/app/migrator/native.go

@@ -15,7 +15,7 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
 const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
@@ -50,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
 		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -75,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -96,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -134,9 +134,9 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
 				return
 			}
-			n := atomic.AddInt64(&comments, 1)
-			if n%1000 == 0 {
-				log.Printf("[DEBUG] imported %d comments", n)
+			num := atomic.AddInt64(&comments, 1)
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
 			}
 		})
 

backend/app/migrator/wordpress.go

@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue

backend/app/notify/notify.go

@@ -29,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment

backend/app/notify/telegram_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NotNil(t, err, "should failed")
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())

backend/app/rest/api/admin.go

@@ -115,14 +115,14 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
+		avatarStore := a.authenticator.AvatarProxy().Store
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
 			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}

backend/app/rest/api/rest.go

@@ -30,6 +30,7 @@ import (
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -44,6 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     *image.Service
 
 	WebRoot         string
 	RemarkURL       string
@@ -219,6 +221,7 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
+			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 
 			ropen.Mount("/rss", s.rssRoutes())
 			ropen.Mount("/img", s.ImageProxy.Routes())
@@ -253,12 +256,25 @@ func (s *Rest) routes() chi.Router {
 			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
 			rauth.Post("/deleteme", s.deleteMeCtrl)
 		})
+
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Post("/picture", s.savePictureCtrl)
+		})
+
 	})
 
 	// respond to /robots.txt with the list of allowed paths
 	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
 		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+				"/img", "/avatar", "/picture"}
 			for i := range allowed {
 				allowed[i] = "Allow: /api/v1" + allowed[i]
 			}

backend/app/rest/api/rest_private.go

@@ -228,14 +228,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments", rest.ErrInternal)
+		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments", rest.ErrInternal)
+		b, errUser := json.Marshal(comments)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
@@ -285,6 +285,31 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
+// POST /image - save image with form request
+func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"id": id})
+}
+
 func (s *Rest) isReadOnly(locator store.Locator) bool {
 	if s.ReadOnlyAge > 0 {
 		// check RO by age

backend/app/rest/api/rest_private_test.go

@@ -1,20 +1,26 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 func TestRest_Create(t *testing.T) {
@@ -488,3 +494,141 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	savePic := func(name string) (id string) {
+		r := strings.NewReader("file content 123")
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, r)
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, "file content 123", string(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+}
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	svc.ImageService = &image.Service{
+		Store: &image.FileSystem{
+			Staging:  "/tmp/remark42/images.staging",
+			Location: "/tmp/remark42/images",
+			MaxSize:  1000,
+		},
+		TTL: time.Millisecond * 100,
+	}
+	svc.DataService.EditDuration = time.Millisecond * 100
+	svc.DataService.ImageService = svc.ImageService
+
+	uploadPicture := func(file, content string) (id string) {
+		r := strings.NewReader(content)
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, r)
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.Contains(t, m["id"], ".png")
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png", "file content 123")
+	id2 := uploadPicture("pic2.png", "file content 12345")
+	id3 := uploadPicture("pic3.png", "file content xyz12365789")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}

backend/app/rest/api/rest_public.go

@@ -1,8 +1,9 @@
 package api
 
 import (
-	"crypto/sha1" //nolint
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -274,12 +275,9 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New() //nolint
-	if _, err := hasher.Write([]byte(k)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls", rest.ErrInternal)
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	h := sha1.Sum([]byte(k)) //nolint
+	sha := base64.URLEncoding.EncodeToString(h[:])
+
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
@@ -330,3 +328,46 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't render posts lits for site %s", siteID)
 	}
 }
+
+// GET /picture/{user}/{id} - get picture
+func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
+	imgRdr, size, err := s.ImageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer imgRdr.Close()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}

backend/app/rest/api/rest_public_test.go

@@ -377,6 +377,37 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 3, pi[1].Count)
 }
 
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
+}
+
 func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -442,5 +473,5 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }

backend/app/rest/api/rest_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -203,6 +204,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 	os.Remove(testDb)
 	os.Remove(testHTML)
 	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
@@ -232,7 +234,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+			},
+			TTL: time.Millisecond * 100,
+		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -258,6 +267,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown

backend/app/rest/api/rss.go

@@ -57,7 +57,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -89,7 +89,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -141,7 +141,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }

backend/app/store/engine/bolt_accessor.go

@@ -155,7 +155,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 
 		return bucket.ForEach(func(k, v []byte) error {
 			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
+			if e = json.Unmarshal(v, &comment); e != nil {
 				return errors.Wrap(e, "failed to unmarshal")
 			}
 			comments = append(comments, comment)
@@ -195,7 +195,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
 			}
 
 			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
+			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
 				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
 				continue
 			}
@@ -335,11 +335,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.
 
 	// retrieve comments for refs
 	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
 		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
+		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
 			comments = append(comments, c)
 		}
 	}

backend/app/store/engine/bolt_admin.go

@@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
 		}
 
 		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
+		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
 			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
+		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
 			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
 		}
 
 		// delete from "last" bucket
 		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
+		if err = lastBkt.Delete([]byte(commentID)); err != nil {
 			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
@@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
 			return nil
 		}
 
-		until, err := time.Parse(tsNano, string(val))
-		if err != nil {
+		until, e := time.Parse(tsNano, string(val))
+		if e != nil {
 			blocked = false
 			return nil
 		}
@@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(blocksBucketName))
 		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
+			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+			if errParse != nil {
+				return errors.Wrap(errParse, "can't parse block ts")
 			}
 			if time.Now().Before(ts) {
 				// get user name from comment user section
 				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
+				userComments, errUser := b.User(siteID, string(k), 1, 0)
+				if errUser == nil && len(userComments) > 0 {
 					userName = userComments[0].User.Name
 				}
 				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})

backend/app/store/engine/mongo.go

@@ -230,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	for _, m := range metas {
-		ids = append(ids, m.ID)
+	for _, meta := range metas {
+		ids = append(ids, meta.ID)
 	}
 	return ids, nil
 }

backend/app/store/image/fs_store.go

@@ -0,0 +1,162 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	uid, err := uuid.NewUUID()
+	if err != nil {
+		return "", errors.Wrap(err, "can't make image uuid")
+	}
+
+	id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	fh, err := os.Create(dst)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't make image file %s", dst)
+	}
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	written, err := io.Copy(fh, lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+	if err = fh.Close(); err != nil {
+		return "", errors.Wrapf(err, "can't close image file %s", dst)
+	}
+	if written > int64(f.MaxSize) {
+		if err = os.Remove(dst); err != nil {
+			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
+		}
+		return "", errors.Errorf("file %s is too large", fileName)
+	}
+	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName)
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", path, age)
+			return os.Remove(path)
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}

backend/app/store/image/fs_store_test.go

@@ -0,0 +1,213 @@
+package image
+
+import (
+	"context"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 5
+	_, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string, content string) (path string) {
+		id, err := svc.Save(file, user, strings.NewReader(content))
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		require.Equal(t, content, string(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1", "blah blah1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1", "blah blah2")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2", "blah blah3")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = &FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    50,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/image.go

@@ -0,0 +1,127 @@
+// Package image handles storing, resizing and retrieval of images
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
+package image
+
+//go:generate sh -c "mockgen -source=image.go -package=image > image_mock.go"
+
+import (
+	"context"
+	"io"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+}
+
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	idsFn func() (ids []string)
+	TS    time.Time
+}
+
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil || s == nil {
+		return
+	}
+
+	s.once.Do(func() {
+		log.Printf("[DEBUG] image submitter activated")
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
+				}
+			}
+			log.Printf("[INFO] image submitter terminated")
+		}()
+	})
+
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
+}
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
+		}
+	}
+}
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}

backend/app/store/image/image_mock.go

@@ -0,0 +1,95 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: image.go
+
+// Package image is a generated GoMock package.
+package image
+
+import (
+	context "context"
+	gomock "github.com/golang/mock/gomock"
+	io "io"
+	reflect "reflect"
+	time "time"
+)
+
+// MockStore is a mock of Store interface
+type MockStore struct {
+	ctrl     *gomock.Controller
+	recorder *MockStoreMockRecorder
+}
+
+// MockStoreMockRecorder is the mock recorder for MockStore
+type MockStoreMockRecorder struct {
+	mock *MockStore
+}
+
+// NewMockStore creates a new mock instance
+func NewMockStore(ctrl *gomock.Controller) *MockStore {
+	mock := &MockStore{ctrl: ctrl}
+	mock.recorder = &MockStoreMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockStore) EXPECT() *MockStoreMockRecorder {
+	return m.recorder
+}
+
+// Save mocks base method
+func (m *MockStore) Save(fileName, userID string, r io.Reader) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Save", fileName, userID, r)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Save indicates an expected call of Save
+func (mr *MockStoreMockRecorder) Save(fileName, userID, r interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockStore)(nil).Save), fileName, userID, r)
+}
+
+// Commit mocks base method
+func (m *MockStore) Commit(id string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Commit", id)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Commit indicates an expected call of Commit
+func (mr *MockStoreMockRecorder) Commit(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockStore)(nil).Commit), id)
+}
+
+// Load mocks base method
+func (m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Load", id)
+	ret0, _ := ret[0].(io.ReadCloser)
+	ret1, _ := ret[1].(int64)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// Load indicates an expected call of Load
+func (mr *MockStoreMockRecorder) Load(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockStore)(nil).Load), id)
+}
+
+// Cleanup mocks base method
+func (m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Cleanup", ctx, ttl)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Cleanup indicates an expected call of Cleanup
+func (mr *MockStoreMockRecorder) Cleanup(ctx, ttl interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockStore)(nil).Cleanup), ctx, ttl)
+}

backend/app/store/image/image_test.go

@@ -0,0 +1,77 @@
+package image
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{ImageAPI: "/blah/"}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
+func TestService_Cleanup(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	store := NewMockStore(ctrl)
+	store.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Times(10)
+
+	svc := Service{Store: store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+}
+
+func TestService_Submit(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	store := NewMockStore(ctrl)
+
+	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	time.Sleep(time.Millisecond * 500)
+}
+
+func TestService_Close(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	store := NewMockStore(ctrl)
+
+	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	svc.Close()
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer func() {
+		ctrl.Finish()
+	}()
+
+	store := NewMockStore(ctrl)
+
+	store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+}

backend/app/store/service/service.go

@@ -12,6 +12,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -28,6 +29,7 @@ type DataStore struct {
 	PositiveScore          bool
 	TitleExtractor         *TitleExtractor
 	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -90,9 +92,32 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
+	s.submitImages(comment)
 	return s.Interface.Create(comment)
 }
 
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			return nil
+		}
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+}
+
 // prepareNewComment sets new comment fields, hashing and sanitizing data
 func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) {
 	// fill ID and time if empty
@@ -129,8 +154,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool)
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
 	comment, err = s.Get(locator, commentID)
@@ -455,8 +480,8 @@ func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
 	return ups, downs
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()

backend/app/store/service/service_test.go

@@ -13,9 +13,12 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/lgr"
+	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -686,6 +689,35 @@ func TestService_Find(t *testing.T) {
 	assert.InDelta(t, 0, res[1].Controversy, 0.01)
 }
 
+func TestService_submitImages(t *testing.T) {
+	defer os.Remove(testDb)
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	mockStore := image.NewMockStore(ctrl)
+	imgSvc := &image.Service{Store: mockStore, TTL: time.Millisecond * 50}
+
+	mockStore.EXPECT().Commit(gomock.Any()).Times(2)
+
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+
+	b.submitImages(c)
+	time.Sleep(250 * time.Millisecond)
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
 	os.Remove(testDb)

backend/go.mod

@@ -20,20 +20,20 @@ require (
 	github.com/go-pkgz/repeater v1.1.1
 	github.com/go-pkgz/rest v1.4.0
 	github.com/go-pkgz/syncs v1.1.0
+	github.com/golang/mock v1.2.0
 	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
 	github.com/gorilla/feeds v1.1.0
-	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
 	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.8.1
 	github.com/rakyll/statik v0.1.3
-	github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16
 	golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0
-	golang.org/x/time v0.0.0-20170927054726-6dc17368e09b // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	gopkg.in/russross/blackfriday.v2 v2.0.0
 )

backend/go.sum

@@ -45,20 +45,20 @@ github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
 github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
 github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
 github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
 github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
 github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -79,9 +79,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
 github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -101,8 +102,8 @@ golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYb
 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

backend/vendor/github.com/golang/mock/AUTHORS

@@ -0,0 +1,12 @@
+# This is the official list of GoMock authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Alex Reece <awreece@gmail.com>
+Google Inc.

backend/vendor/github.com/golang/mock/CONTRIBUTORS

@@ -0,0 +1,37 @@
+# This is the official list of people who can contribute (and typically
+# have contributed) code to the gomock repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# An entry with two email addresses specifies that the
+# first address should be used in the submit logs and
+# that the second address should be recognized as the
+# same person when interacting with Rietveld.
+
+# Please keep the list sorted.
+
+Aaron Jacobs <jacobsa@google.com> <aaronjjacobs@gmail.com>
+Alex Reece <awreece@gmail.com>
+David Symonds <dsymonds@golang.org>
+Ryan Barrett <ryanb@google.com>

backend/vendor/github.com/golang/mock/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

backend/vendor/github.com/golang/mock/gomock/call.go

@@ -0,0 +1,420 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// Call represents an expected call to a mock.
+type Call struct {
+	t TestHelper // for triggering test failures on invalid call setup
+
+	receiver   interface{}  // the receiver of the method call
+	method     string       // the name of the method
+	methodType reflect.Type // the type of the method
+	args       []Matcher    // the args
+	origin     string       // file and line number of call setup
+
+	preReqs []*Call // prerequisite calls
+
+	// Expectations
+	minCalls, maxCalls int
+
+	numCalls int // actual number made
+
+	// actions are called when this Call is called. Each action gets the args and
+	// can set the return values by returning a non-nil slice. Actions run in the
+	// order they are created.
+	actions []func([]interface{}) []interface{}
+}
+
+// newCall creates a *Call. It requires the method type in order to support
+// unexported methods.
+func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	t.Helper()
+
+	// TODO: check arity, types.
+	margs := make([]Matcher, len(args))
+	for i, arg := range args {
+		if m, ok := arg.(Matcher); ok {
+			margs[i] = m
+		} else if arg == nil {
+			// Handle nil specially so that passing a nil interface value
+			// will match the typed nils of concrete args.
+			margs[i] = Nil()
+		} else {
+			margs[i] = Eq(arg)
+		}
+	}
+
+	origin := callerInfo(3)
+	actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
+		// Synthesize the zero value for each of the return args' types.
+		rets := make([]interface{}, methodType.NumOut())
+		for i := 0; i < methodType.NumOut(); i++ {
+			rets[i] = reflect.Zero(methodType.Out(i)).Interface()
+		}
+		return rets
+	}}
+	return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
+		args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
+}
+
+// AnyTimes allows the expectation to be called 0 or more times
+func (c *Call) AnyTimes() *Call {
+	c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity
+	return c
+}
+
+// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also
+// sets the maximum number of calls to infinity.
+func (c *Call) MinTimes(n int) *Call {
+	c.minCalls = n
+	if c.maxCalls == 1 {
+		c.maxCalls = 1e8
+	}
+	return c
+}
+
+// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also
+// sets the minimum number of calls to 0.
+func (c *Call) MaxTimes(n int) *Call {
+	c.maxCalls = n
+	if c.minCalls == 1 {
+		c.minCalls = 0
+	}
+	return c
+}
+
+// DoAndReturn declares the action to run when the call is matched.
+// The return values from this function are returned by the mocked function.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) DoAndReturn(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		vrets := v.Call(vargs)
+		rets := make([]interface{}, len(vrets))
+		for i, ret := range vrets {
+			rets[i] = ret.Interface()
+		}
+		return rets
+	})
+	return c
+}
+
+// Do declares the action to run when the call is matched. The function's
+// return values are ignored to retain backward compatibility. To use the
+// return values call DoAndReturn.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) Do(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		v.Call(vargs)
+		return nil
+	})
+	return c
+}
+
+// Return declares the values to be returned by the mocked function call.
+func (c *Call) Return(rets ...interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	if len(rets) != mt.NumOut() {
+		c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]",
+			c.receiver, c.method, len(rets), mt.NumOut(), c.origin)
+	}
+	for i, ret := range rets {
+		if got, want := reflect.TypeOf(ret), mt.Out(i); got == want {
+			// Identical types; nothing to do.
+		} else if got == nil {
+			// Nil needs special handling.
+			switch want.Kind() {
+			case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+				// ok
+			default:
+				c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]",
+					i, c.receiver, c.method, want, c.origin)
+			}
+		} else if got.AssignableTo(want) {
+			// Assignable type relation. Make the assignment now so that the generated code
+			// can return the values with a type assertion.
+			v := reflect.New(want).Elem()
+			v.Set(reflect.ValueOf(ret))
+			rets[i] = v.Interface()
+		} else {
+			c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]",
+				i, c.receiver, c.method, got, want, c.origin)
+		}
+	}
+
+	c.addAction(func([]interface{}) []interface{} {
+		return rets
+	})
+
+	return c
+}
+
+// Times declares the exact number of times a function call is expected to be executed.
+func (c *Call) Times(n int) *Call {
+	c.minCalls, c.maxCalls = n, n
+	return c
+}
+
+// SetArg declares an action that will set the nth argument's value,
+// indirected through a pointer. Or, in the case of a slice, SetArg
+// will copy value's elements into the nth argument.
+func (c *Call) SetArg(n int, value interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	// TODO: This will break on variadic methods.
+	// We will need to check those at invocation time.
+	if n < 0 || n >= mt.NumIn() {
+		c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]",
+			n, mt.NumIn(), c.origin)
+	}
+	// Permit setting argument through an interface.
+	// In the interface case, we don't (nay, can't) check the type here.
+	at := mt.In(n)
+	switch at.Kind() {
+	case reflect.Ptr:
+		dt := at.Elem()
+		if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) {
+			c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]",
+				n, vt, dt, c.origin)
+		}
+	case reflect.Interface:
+		// nothing to do
+	case reflect.Slice:
+		// nothing to do
+	default:
+		c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]",
+			n, at, c.origin)
+	}
+
+	c.addAction(func(args []interface{}) []interface{} {
+		v := reflect.ValueOf(value)
+		switch reflect.TypeOf(args[n]).Kind() {
+		case reflect.Slice:
+			setSlice(args[n], v)
+		default:
+			reflect.ValueOf(args[n]).Elem().Set(v)
+		}
+		return nil
+	})
+	return c
+}
+
+// isPreReq returns true if other is a direct or indirect prerequisite to c.
+func (c *Call) isPreReq(other *Call) bool {
+	for _, preReq := range c.preReqs {
+		if other == preReq || preReq.isPreReq(other) {
+			return true
+		}
+	}
+	return false
+}
+
+// After declares that the call may only match after preReq has been exhausted.
+func (c *Call) After(preReq *Call) *Call {
+	c.t.Helper()
+
+	if c == preReq {
+		c.t.Fatalf("A call isn't allowed to be its own prerequisite")
+	}
+	if preReq.isPreReq(c) {
+		c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq)
+	}
+
+	c.preReqs = append(c.preReqs, preReq)
+	return c
+}
+
+// Returns true if the minimum number of calls have been made.
+func (c *Call) satisfied() bool {
+	return c.numCalls >= c.minCalls
+}
+
+// Returns true iff the maximum number of calls have been made.
+func (c *Call) exhausted() bool {
+	return c.numCalls >= c.maxCalls
+}
+
+func (c *Call) String() string {
+	args := make([]string, len(c.args))
+	for i, arg := range c.args {
+		args[i] = arg.String()
+	}
+	arguments := strings.Join(args, ", ")
+	return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin)
+}
+
+// Tests if the given call matches the expected call.
+// If yes, returns nil. If no, returns error with message explaining why it does not match.
+func (c *Call) matches(args []interface{}) error {
+	if !c.methodType.IsVariadic() {
+		if len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+
+		for i, m := range c.args {
+			if !m.Matches(args[i]) {
+				return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+					c.origin, strconv.Itoa(i), args[i], m)
+			}
+		}
+	} else {
+		if len(c.args) < c.methodType.NumIn()-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d",
+				c.origin, len(c.args), c.methodType.NumIn()-1)
+		}
+		if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+		if len(args) < len(c.args)-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d",
+				c.origin, len(args), len(c.args)-1)
+		}
+
+		for i, m := range c.args {
+			if i < c.methodType.NumIn()-1 {
+				// Non-variadic args
+				if !m.Matches(args[i]) {
+					return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+						c.origin, strconv.Itoa(i), args[i], m)
+				}
+				continue
+			}
+			// The last arg has a possibility of a variadic argument, so let it branch
+
+			// sample: Foo(a int, b int, c ...int)
+			if i < len(c.args) && i < len(args) {
+				if m.Matches(args[i]) {
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any())
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher)
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC)
+					// Got Foo(a, b) want Foo(matcherA, matcherB)
+					// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD)
+					continue
+				}
+			}
+
+			// The number of actual args don't match the number of matchers,
+			// or the last matcher is a slice and the last arg is not.
+			// If this function still matches it is because the last matcher
+			// matches all the remaining arguments or the lack of any.
+			// Convert the remaining arguments, if any, into a slice of the
+			// expected type.
+			vargsType := c.methodType.In(c.methodType.NumIn() - 1)
+			vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
+			for _, arg := range args[i:] {
+				vargs = reflect.Append(vargs, reflect.ValueOf(arg))
+			}
+			if m.Matches(vargs.Interface()) {
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
+				// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher)
+				break
+			}
+			// Wrong number of matchers or not match. Fail.
+			// Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
+			// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB)
+			return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+				c.origin, strconv.Itoa(i), args[i:], c.args[i])
+
+		}
+	}
+
+	// Check that all prerequisite calls have been satisfied.
+	for _, preReqCall := range c.preReqs {
+		if !preReqCall.satisfied() {
+			return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
+				c.origin, preReqCall, c)
+		}
+	}
+
+	// Check that the call is not exhausted.
+	if c.exhausted() {
+		return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin)
+	}
+
+	return nil
+}
+
+// dropPrereqs tells the expected Call to not re-check prerequisite calls any
+// longer, and to return its current set.
+func (c *Call) dropPrereqs() (preReqs []*Call) {
+	preReqs = c.preReqs
+	c.preReqs = nil
+	return
+}
+
+func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} {
+	c.numCalls++
+	return c.actions
+}
+
+// InOrder declares that the given calls should occur in order.
+func InOrder(calls ...*Call) {
+	for i := 1; i < len(calls); i++ {
+		calls[i].After(calls[i-1])
+	}
+}
+
+func setSlice(arg interface{}, v reflect.Value) {
+	va := reflect.ValueOf(arg)
+	for i := 0; i < v.Len(); i++ {
+		va.Index(i).Set(v.Index(i))
+	}
+}
+
+func (c *Call) addAction(action func([]interface{}) []interface{}) {
+	c.actions = append(c.actions, action)
+}

backend/vendor/github.com/golang/mock/gomock/callset.go

@@ -0,0 +1,108 @@
+// Copyright 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// callSet represents a set of expected calls, indexed by receiver and method
+// name.
+type callSet struct {
+	// Calls that are still expected.
+	expected map[callSetKey][]*Call
+	// Calls that have been exhausted.
+	exhausted map[callSetKey][]*Call
+}
+
+// callSetKey is the key in the maps in callSet
+type callSetKey struct {
+	receiver interface{}
+	fname    string
+}
+
+func newCallSet() *callSet {
+	return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)}
+}
+
+// Add adds a new expected call.
+func (cs callSet) Add(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	m := cs.expected
+	if call.exhausted() {
+		m = cs.exhausted
+	}
+	m[key] = append(m[key], call)
+}
+
+// Remove removes an expected call.
+func (cs callSet) Remove(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	calls := cs.expected[key]
+	for i, c := range calls {
+		if c == call {
+			// maintain order for remaining calls
+			cs.expected[key] = append(calls[:i], calls[i+1:]...)
+			cs.exhausted[key] = append(cs.exhausted[key], call)
+			break
+		}
+	}
+}
+
+// FindMatch searches for a matching call. Returns error with explanation message if no call matched.
+func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) {
+	key := callSetKey{receiver, method}
+
+	// Search through the expected calls.
+	expected := cs.expected[key]
+	var callsErrors bytes.Buffer
+	for _, call := range expected {
+		err := call.matches(args)
+		if err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		} else {
+			return call, nil
+		}
+	}
+
+	// If we haven't found a match then search through the exhausted calls so we
+	// get useful error messages.
+	exhausted := cs.exhausted[key]
+	for _, call := range exhausted {
+		if err := call.matches(args); err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		}
+	}
+
+	if len(expected)+len(exhausted) == 0 {
+		fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
+	}
+
+	return nil, fmt.Errorf(callsErrors.String())
+}
+
+// Failures returns the calls that are not satisfied.
+func (cs callSet) Failures() []*Call {
+	failures := make([]*Call, 0, len(cs.expected))
+	for _, calls := range cs.expected {
+		for _, call := range calls {
+			if !call.satisfied() {
+				failures = append(failures, call)
+			}
+		}
+	}
+	return failures
+}

backend/vendor/github.com/golang/mock/gomock/controller.go

@@ -0,0 +1,235 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// GoMock - a mock framework for Go.
+//
+// Standard usage:
+//   (1) Define an interface that you wish to mock.
+//         type MyInterface interface {
+//           SomeMethod(x int64, y string)
+//         }
+//   (2) Use mockgen to generate a mock from the interface.
+//   (3) Use the mock in a test:
+//         func TestMyThing(t *testing.T) {
+//           mockCtrl := gomock.NewController(t)
+//           defer mockCtrl.Finish()
+//
+//           mockObj := something.NewMockMyInterface(mockCtrl)
+//           mockObj.EXPECT().SomeMethod(4, "blah")
+//           // pass mockObj to a real object and play with it.
+//         }
+//
+// By default, expected calls are not enforced to run in any particular order.
+// Call order dependency can be enforced by use of InOrder and/or Call.After.
+// Call.After can create more varied call order dependencies, but InOrder is
+// often more convenient.
+//
+// The following examples create equivalent call order dependencies.
+//
+// Example of using Call.After to chain expected call order:
+//
+//     firstCall := mockObj.EXPECT().SomeMethod(1, "first")
+//     secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall)
+//     mockObj.EXPECT().SomeMethod(3, "third").After(secondCall)
+//
+// Example of using InOrder to declare expected call order:
+//
+//     gomock.InOrder(
+//         mockObj.EXPECT().SomeMethod(1, "first"),
+//         mockObj.EXPECT().SomeMethod(2, "second"),
+//         mockObj.EXPECT().SomeMethod(3, "third"),
+//     )
+//
+// TODO:
+//	- Handle different argument/return types (e.g. ..., chan, map, interface).
+package gomock
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"runtime"
+	"sync"
+)
+
+// A TestReporter is something that can be used to report test failures.
+// It is satisfied by the standard library's *testing.T.
+type TestReporter interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+
+// TestHelper is a TestReporter that has the Helper method.  It is satisfied
+// by the standard library's *testing.T.
+type TestHelper interface {
+	TestReporter
+	Helper()
+}
+
+// A Controller represents the top-level control of a mock ecosystem.
+// It defines the scope and lifetime of mock objects, as well as their expectations.
+// It is safe to call Controller's methods from multiple goroutines.
+type Controller struct {
+	// T should only be called within a generated mock. It is not intended to
+	// be used in user code and may be changed in future versions. T is the
+	// TestReporter passed in when creating the Controller via NewController.
+	// If the TestReporter does not implment a TestHelper it will be wrapped
+	// with a nopTestHelper.
+	T             TestHelper
+	mu            sync.Mutex
+	expectedCalls *callSet
+	finished      bool
+}
+
+func NewController(t TestReporter) *Controller {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	return &Controller{
+		T:             h,
+		expectedCalls: newCallSet(),
+	}
+}
+
+type cancelReporter struct {
+	TestHelper
+	cancel func()
+}
+
+func (r *cancelReporter) Errorf(format string, args ...interface{}) {
+	r.TestHelper.Errorf(format, args...)
+}
+func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
+	defer r.cancel()
+	r.TestHelper.Fatalf(format, args...)
+}
+
+// WithContext returns a new Controller and a Context, which is cancelled on any
+// fatal failure.
+func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	return NewController(&cancelReporter{h, cancel}), ctx
+}
+
+type nopTestHelper struct {
+	TestReporter
+}
+
+func (h nopTestHelper) Helper() {}
+
+func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	recv := reflect.ValueOf(receiver)
+	for i := 0; i < recv.Type().NumMethod(); i++ {
+		if recv.Type().Method(i).Name == method {
+			return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...)
+		}
+	}
+	ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver)
+	panic("unreachable")
+}
+
+func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	call := newCall(ctrl.T, receiver, method, methodType, args...)
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+	ctrl.expectedCalls.Add(call)
+
+	return call
+}
+
+func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} {
+	ctrl.T.Helper()
+
+	// Nest this code so we can use defer to make sure the lock is released.
+	actions := func() []func([]interface{}) []interface{} {
+		ctrl.T.Helper()
+		ctrl.mu.Lock()
+		defer ctrl.mu.Unlock()
+
+		expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
+		if err != nil {
+			origin := callerInfo(2)
+			ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
+		}
+
+		// Two things happen here:
+		// * the matching call no longer needs to check prerequite calls,
+		// * and the prerequite calls are no longer expected, so remove them.
+		preReqCalls := expected.dropPrereqs()
+		for _, preReqCall := range preReqCalls {
+			ctrl.expectedCalls.Remove(preReqCall)
+		}
+
+		actions := expected.call(args)
+		if expected.exhausted() {
+			ctrl.expectedCalls.Remove(expected)
+		}
+		return actions
+	}()
+
+	var rets []interface{}
+	for _, action := range actions {
+		if r := action(args); r != nil {
+			rets = r
+		}
+	}
+
+	return rets
+}
+
+func (ctrl *Controller) Finish() {
+	ctrl.T.Helper()
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+
+	if ctrl.finished {
+		ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
+	}
+	ctrl.finished = true
+
+	// If we're currently panicking, probably because this is a deferred call,
+	// pass through the panic.
+	if err := recover(); err != nil {
+		panic(err)
+	}
+
+	// Check that all remaining expected calls are satisfied.
+	failures := ctrl.expectedCalls.Failures()
+	for _, call := range failures {
+		ctrl.T.Errorf("missing call(s) to %v", call)
+	}
+	if len(failures) != 0 {
+		ctrl.T.Fatalf("aborting test due to missing call(s)")
+	}
+}
+
+func callerInfo(skip int) string {
+	if _, file, line, ok := runtime.Caller(skip + 1); ok {
+		return fmt.Sprintf("%s:%d", file, line)
+	}
+	return "unknown file"
+}

backend/vendor/github.com/golang/mock/gomock/matchers.go

@@ -0,0 +1,122 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// A Matcher is a representation of a class of values.
+// It is used to represent the valid or expected arguments to a mocked method.
+type Matcher interface {
+	// Matches returns whether x is a match.
+	Matches(x interface{}) bool
+
+	// String describes what the matcher matches.
+	String() string
+}
+
+type anyMatcher struct{}
+
+func (anyMatcher) Matches(x interface{}) bool {
+	return true
+}
+
+func (anyMatcher) String() string {
+	return "is anything"
+}
+
+type eqMatcher struct {
+	x interface{}
+}
+
+func (e eqMatcher) Matches(x interface{}) bool {
+	return reflect.DeepEqual(e.x, x)
+}
+
+func (e eqMatcher) String() string {
+	return fmt.Sprintf("is equal to %v", e.x)
+}
+
+type nilMatcher struct{}
+
+func (nilMatcher) Matches(x interface{}) bool {
+	if x == nil {
+		return true
+	}
+
+	v := reflect.ValueOf(x)
+	switch v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map,
+		reflect.Ptr, reflect.Slice:
+		return v.IsNil()
+	}
+
+	return false
+}
+
+func (nilMatcher) String() string {
+	return "is nil"
+}
+
+type notMatcher struct {
+	m Matcher
+}
+
+func (n notMatcher) Matches(x interface{}) bool {
+	return !n.m.Matches(x)
+}
+
+func (n notMatcher) String() string {
+	// TODO: Improve this if we add a NotString method to the Matcher interface.
+	return "not(" + n.m.String() + ")"
+}
+
+type assignableToTypeOfMatcher struct {
+	targetType reflect.Type
+}
+
+func (m assignableToTypeOfMatcher) Matches(x interface{}) bool {
+	return reflect.TypeOf(x).AssignableTo(m.targetType)
+}
+
+func (m assignableToTypeOfMatcher) String() string {
+	return "is assignable to " + m.targetType.Name()
+}
+
+// Constructors
+func Any() Matcher             { return anyMatcher{} }
+func Eq(x interface{}) Matcher { return eqMatcher{x} }
+func Nil() Matcher             { return nilMatcher{} }
+func Not(x interface{}) Matcher {
+	if m, ok := x.(Matcher); ok {
+		return notMatcher{m}
+	}
+	return notMatcher{Eq(x)}
+}
+
+// AssignableToTypeOf is a Matcher that matches if the parameter to the mock
+// function is assignable to the type of the parameter to this function.
+//
+// Example usage:
+//
+// 		dbMock.EXPECT().
+// 			Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})).
+// 			Return(errors.New("DB error"))
+//
+func AssignableToTypeOf(x interface{}) Matcher {
+	return assignableToTypeOfMatcher{reflect.TypeOf(x)}
+}

backend/vendor/github.com/hashicorp/errwrap/README.md

@@ -48,7 +48,7 @@ func main() {
 	// We can use the Contains helpers to check if an error contains
 	// another error. It is safe to do this with a nil error, or with
 	// an error that doesn't even use the errwrap package.
-	if errwrap.Contains(err, ErrNotExist) {
+	if errwrap.Contains(err, "does not exist") {
 		// Do something
 	}
 	if errwrap.ContainsType(err, new(os.PathError)) {

backend/vendor/github.com/hashicorp/errwrap/go.mod

@@ -0,0 +1 @@
+module github.com/hashicorp/errwrap

backend/vendor/github.com/hashicorp/golang-lru/lru.go

@@ -40,35 +40,31 @@ func (c *Cache) Purge() {
 // Add adds a value to the cache.  Returns true if an eviction occurred.
 func (c *Cache) Add(key, value interface{}) (evicted bool) {
 	c.lock.Lock()
-	evicted = c.lru.Add(key, value)
-	c.lock.Unlock()
-	return evicted
+	defer c.lock.Unlock()
+	return c.lru.Add(key, value)
 }
 
 // Get looks up a key's value from the cache.
 func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
 	c.lock.Lock()
-	value, ok = c.lru.Get(key)
-	c.lock.Unlock()
-	return value, ok
+	defer c.lock.Unlock()
+	return c.lru.Get(key)
 }
 
 // Contains checks if a key is in the cache, without updating the
 // recent-ness or deleting it for being stale.
 func (c *Cache) Contains(key interface{}) bool {
 	c.lock.RLock()
-	containKey := c.lru.Contains(key)
-	c.lock.RUnlock()
-	return containKey
+	defer c.lock.RUnlock()
+	return c.lru.Contains(key)
 }
 
 // Peek returns the key value (or undefined if not found) without updating
 // the "recently used"-ness of the key.
 func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) {
 	c.lock.RLock()
-	value, ok = c.lru.Peek(key)
-	c.lock.RUnlock()
-	return value, ok
+	defer c.lock.RUnlock()
+	return c.lru.Peek(key)
 }
 
 // ContainsOrAdd checks if a key is in the cache  without updating the
@@ -102,15 +98,13 @@ func (c *Cache) RemoveOldest() {
 // Keys returns a slice of the keys in the cache, from oldest to newest.
 func (c *Cache) Keys() []interface{} {
 	c.lock.RLock()
-	keys := c.lru.Keys()
-	c.lock.RUnlock()
-	return keys
+	defer c.lock.RUnlock()
+	return c.lru.Keys()
 }
 
 // Len returns the number of items in the cache.
 func (c *Cache) Len() int {
 	c.lock.RLock()
-	length := c.lru.Len()
-	c.lock.RUnlock()
-	return length
+	defer c.lock.RUnlock()
+	return c.lru.Len()
 }

backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod

@@ -0,0 +1 @@
+module github.com/shurcooL/sanitized_anchor_name

backend/vendor/golang.org/x/time/rate/rate.go

@@ -6,6 +6,7 @@
 package rate
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"sync"
@@ -212,19 +213,8 @@ func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation {
 	return &r
 }
 
-// contextContext is a temporary(?) copy of the context.Context type
-// to support both Go 1.6 using golang.org/x/net/context and Go 1.7+
-// with the built-in context package. If people ever stop using Go 1.6
-// we can remove this.
-type contextContext interface {
-	Deadline() (deadline time.Time, ok bool)
-	Done() <-chan struct{}
-	Err() error
-	Value(key interface{}) interface{}
-}
-
 // Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) wait(ctx contextContext) (err error) {
+func (lim *Limiter) Wait(ctx context.Context) (err error) {
 	return lim.WaitN(ctx, 1)
 }
 
@@ -232,7 +222,7 @@ func (lim *Limiter) wait(ctx contextContext) (err error) {
 // It returns an error if n exceeds the Limiter's burst size, the Context is
 // canceled, or the expected wait time exceeds the Context's Deadline.
 // The burst limit is ignored if the rate limit is Inf.
-func (lim *Limiter) waitN(ctx contextContext, n int) (err error) {
+func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
 	if n > lim.burst && lim.limit != Inf {
 		return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, lim.burst)
 	}
@@ -253,8 +243,12 @@ func (lim *Limiter) waitN(ctx contextContext, n int) (err error) {
 	if !r.ok {
 		return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n)
 	}
-	// Wait
-	t := time.NewTimer(r.DelayFrom(now))
+	// Wait if necessary
+	delay := r.DelayFrom(now)
+	if delay == 0 {
+		return nil
+	}
+	t := time.NewTimer(delay)
 	defer t.Stop()
 	select {
 	case <-t.C:

backend/vendor/golang.org/x/time/rate/rate_go16.go

@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !go1.7
-
-package rate
-
-import "golang.org/x/net/context"
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.waitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	return lim.waitN(ctx, n)
-}

backend/vendor/golang.org/x/time/rate/rate_go17.go

@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.7
-
-package rate
-
-import "context"
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.waitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	return lim.waitN(ctx, n)
-}

backend/vendor/modules.txt

@@ -52,17 +52,19 @@ github.com/go-pkgz/rest
 github.com/go-pkgz/rest/logger
 # github.com/go-pkgz/syncs v1.1.0
 github.com/go-pkgz/syncs
+# github.com/golang/mock v1.2.0
+github.com/golang/mock/gomock
 # github.com/golang/protobuf v1.2.0
 github.com/golang/protobuf/proto
 # github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
 github.com/google/uuid
 # github.com/gorilla/feeds v1.1.0
 github.com/gorilla/feeds
-# github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce
+# github.com/hashicorp/errwrap v1.0.0
 github.com/hashicorp/errwrap
 # github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
 github.com/hashicorp/go-multierror
-# github.com/hashicorp/golang-lru v0.5.1
+# github.com/hashicorp/golang-lru v0.5.0
 github.com/hashicorp/golang-lru
 github.com/hashicorp/golang-lru/simplelru
 # github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
@@ -79,7 +81,7 @@ github.com/pkg/errors
 github.com/pmezard/go-difflib/difflib
 # github.com/rakyll/statik v0.1.3
 github.com/rakyll/statik/fs
-# github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95
+# github.com/shurcooL/sanitized_anchor_name v1.0.0
 github.com/shurcooL/sanitized_anchor_name
 # github.com/stretchr/testify v1.3.0
 github.com/stretchr/testify/assert
@@ -106,7 +108,7 @@ golang.org/x/oauth2/jws
 golang.org/x/oauth2/jwt
 # golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb
 golang.org/x/sys/unix
-# golang.org/x/time v0.0.0-20170927054726-6dc17368e09b
+# golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 golang.org/x/time/rate
 # google.golang.org/appengine v1.4.0
 google.golang.org/appengine