backport url sanitizer to 1.6

change golangci-lint format to github-actions for CI

.github/FUNDING.yml

@@ -1,4 +1,4 @@
 # These are supported funding model platforms
 
 github: [umputun]
-#patreon: remark42
+patreon: remark42

.github/workflows/ci-test-backend.yml

@@ -8,13 +8,11 @@ on:
       - '.github/workflows/ci-test-backend.yml'
       - 'backend/**'
       - '!backend/scripts/**'
-      - '.golangci.yml'
   pull_request:
     paths:
       - '.github/workflows/ci-test-backend.yml'
       - 'backend/**'
       - '!backend/scripts/**'
-      - '.golangci.yml'
 
 jobs:
   backend:
@@ -24,41 +22,39 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: debug if needed
-        run: |
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
-          export DEBUG=${DEBUG:-false}
-          if [[ "$DEBUG" == "true" ]]; then
-            env
-          fi
         env:
           DEBUG: ${{secrets.DEBUG}}
 
       - name: install go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.13
+          go-version: 1.14
-
-      - name: test backend
-        run: |
-          export TZ="America/Chicago"
-          date
-          cd backend/app
-          go test -mod=vendor -timeout=60s -covermode=count -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
-          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
-          cd ../_example/memory_store
-          go test -race ./...
 
       - name: install golangci-lint and goveralls
         run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.20.0
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.25.0
           go get -u github.com/mattn/goveralls
 
-      - name: run backend linters
+      - name: test and lint backend
-        run: $GITHUB_WORKSPACE/golangci-lint run --config .golangci.yml ./...
+        run: |
-        working-directory: backend
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          GOFLAGS: "-mod=vendor"
+          TZ: "America/Chicago"
 
-      - name: run linters for examples
+      - name: test and lint examples
-        run: $GITHUB_WORKSPACE/golangci-lint run --config ${GITHUB_WORKSPACE}/backend/.golangci.yml ./...
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
         working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
 
       - name: submit coverage
         run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov

.github/workflows/ci-test-frontend.yml

@@ -17,12 +17,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
       - uses: actions/setup-node@v1
 
-      - run: npm ci
+      - run: npm ci --loglevel warn
         working-directory: ./frontend
 
-      - run: npx run-p check lint test
+      - run: npx run-p check lint
         working-directory: ./frontend

.gitignore

@@ -19,6 +19,7 @@ debug.test
 remark42
 /bin/
 /backend/var/
+/backend/app/var/
 compose-private-backend.yml
 compose-private-frontend.yml
 compose-private.yml

.golangci.yml

@@ -1,2 +0,0 @@
-service:
-  golangci-lint-version: 1.19.1

Dockerfile

@@ -20,7 +20,7 @@ ENV GOFLAGS="-mod=vendor"
 RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
         golangci-lint run --config ../.golangci.yml ./... ; \
     else echo "skip backend tests and linter" ; fi
@@ -40,7 +40,7 @@ ENV HUSKY_SKIP_INSTALL=true
 RUN apk add --no-cache --update git
 ADD frontend/package.json /srv/frontend/package.json
 ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/frontend && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci --loglevel warn
 
 FROM node:10.11-alpine as build-frontend
 
@@ -51,7 +51,7 @@ ARG NODE_ENV=production
 COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
 ADD frontend /srv/frontend
 RUN cd /srv/frontend && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p check lint test build ; \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 

Dockerfile.artifacts

@@ -45,7 +45,6 @@ COPY --from=build-frontend /srv/frontend/public/ web
 RUN \
     export WEB_ROOT=/build/backend/web && \
     sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
-    sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
     statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
     ls -la /build/backend/app/rest/api/statik.go && \
     ls -la /build/backend/web/

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2020 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -42,7 +42,7 @@ For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki
   - [Install](#install)
     - [Backend](#backend)
       - [With Docker](#with-docker)
-      - [Without docker](#without-docker)
+      - [Without Docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
       - [Quick installation test](#quick-installation-test)
@@ -89,15 +89,15 @@ _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
 * make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
-* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
+* pull prepared images from the DockerHub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
-#### Without docker
+#### Without Docker
 
 * download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
 * unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
 * run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
-* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
+* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
 
 #### Parameters
 
@@ -155,6 +155,7 @@ _this is the recommended way to run remark42_
 | notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
 | notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
 | notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
+| notify.email.notify_admin | NOTIFY_EMAIL_ADMIN    | `false`                  | notify admin on new comments via ADMIN_SHARED_EMAIL |
 | smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
 | smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
 | smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
@@ -363,14 +364,14 @@ It will expand login info and show full user ID.
 
 #### Docker parameters
 
-Two parameters allow to customize docker container on the system level:
+Two parameters allow customizing Docker container on the system level:
 
 - `APP_UID` - sets UID to run remark42 application in container (default=1001)
 - `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
 
 _see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
 
-example of compose:
+example of `docker-compose.yml`:
 
 ```yaml
 version: '2'
@@ -434,7 +435,8 @@ Add this snippet to the bottom of web page:
                      // in well defined order
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
-    page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
+    page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
+    locale: 'en' // set up locale and language, if it isn't defined default value ('en') will be used
   };
 
   (function(c) {
@@ -469,6 +471,13 @@ Just call this function and pass a name of the theme that you want to turn on:
 window.REMARK42.changeTheme('light');
 ```
 
+##### Locales
+
+Right now Remark is translated to en, ru (partially), de, and fi languages.
+You can pick one using [configuration object](#setup-on-your-website).
+
+Do you want translate remark42 to other locale? Please see [this documentation](https://github.com/umputun/remark42/blob/master/docs/translation.md) for details.
+
 #### Last comments
 
 It's a widget which renders list of last comments from your site.
@@ -541,13 +550,13 @@ Also script can use `url` property from `remark_config` object, or `window.locat
 
 ## Build from the source
 
-- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
+- to build Docker container - `make docker`. This command will produce container `umputun/remark42`.
 - to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
  `remark42` file with everything embedded.
 
 ## Development
 
-You can use fully functional local version to develop and test both frontend & backend.
+You can use fully functional local version to develop and test both frontend & backend. It requires at least 2GB RAM or swap enabled
 
 To bring it up run:
 
@@ -565,29 +574,34 @@ To access UI demo page go to `127.0.0.1:8080/web`.
 By default, you would be logged in as `dev_user` which defined as admin.
 You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
 
-Backend docker compose config by default skips running frontend related tests.
+Backend Docker Compose config by default skips running frontend related tests.
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
+Frontend Docker Compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
 
 ### Backend development
 
-In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
+In order to run backend locally (development mode, without Docker) you have to have the latest stable `go` toolchain [installed](https://golang.org/doc/install).
 
-To run backend - `go run backend/app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
+To run backend - `cd backend; go run app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
 It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
 `HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
 
 ### Frontend development
 
-#### Build
+#### Developer guide
 
-* install [Node.js 8](https://nodejs.org/en/) or higher;
+Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
-* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
+
+#### Build
+You should have at least 2GB RAM or swap enabled for building
+
+* install [Node.js 12.11](https://nodejs.org/en/) or higher;
+* install [NPM 6.13.4](https://www.npmjs.com/package/npm);
 * run `npm install` inside `./frontend`;
 * run `npm run build` there;
 * result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
-It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
+It used to reformat your frontend code using `prettier` and lint with `eslint` and `stylelint` before every commit.
 
 #### Devserver
 
@@ -602,12 +616,17 @@ You can attach to locally running backend by providing `REMARK_URL` environment
 npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 ```
 
+The best way for start local developer enviroment:
+```sh
+cp compose-dev-frontend.yml compose-private-frontend.yml
+docker-compose -f compose-private-frontend.yml up --build
+cd frontend
+npm run dev
+```
+
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
-#### Frontend guide
-
-Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
 
 ## API
 
@@ -860,5 +879,6 @@ _all admin calls require auth and admin privilege_
 * User can edit comments in 5 mins (configurable) window after creation.
 * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
 * All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
-* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
+* Images can be proxied (`IMAGE_PROXY_HTTP2HTTPS=true`) to prevent mixed http/https.
+* All images can be proxied and saved (`IMAGE_PROXY_CACHE_EXTERNAL=true`) instead of serving from original location. Beware, images which are posted with this parameter enabled will be served from proxy even after it will be disabled.
 * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.

backend/.golangci.yml

@@ -1,11 +1,36 @@
 run:
-  tests: false
   output:
     format: tab
   skip-dirs:
     - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0.1
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+
 linters:
   enable:
+    - megacheck
+    - golint
+    - govet
     - unconvert
     - megacheck
     - structcheck
@@ -19,4 +44,28 @@ linters:
     - typecheck
     - ineffassign
     - varcheck
+    - stylecheck
+    - gochecknoinits
+    - scopelint
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
   disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "should have a package comment, unless it's in another file for this package"
+      linters:
+        - golint
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false
+
+service:
+  golangci-lint-version: 1.23.x

backend/_example/memory_store/README.md

@@ -4,10 +4,10 @@
 
 In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
 
-1. docker-compose -f compose-dev-memstore.yml build
+1. `docker-compose -f compose-dev-memstore.yml build`
-1. docker-compose -f compose-dev-memstore.yml up
+1. `docker-compose -f compose-dev-memstore.yml up`
 
 As usual, demo site will run on http://127.0.0.1:8080/web/
 
-note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
-. In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 
+In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -9,6 +9,7 @@ package accessor
 import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+
 	"github.com/umputun/remark/backend/app/store/admin"
 )
 
@@ -74,7 +75,7 @@ func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
 		return errors.Errorf("site %s not found", siteID)
 	}
 	if ev == admin.EvCreate {
-		resp.CountCreated += 1 // not a good idea, just for demo
+		resp.CountCreated++ // not a good idea, just for demo
 		m.data[siteID] = resp
 	}
 	return nil

backend/_example/memory_store/accessor/admin_test.go

@@ -47,9 +47,11 @@ func TestMemAdmin_Get(t *testing.T) {
 
 	admins, err = ms.Admins("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
 
 	email, err = ms.Email("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
 
 	enabled, err := ms.Enabled("site1")
 	assert.NoError(t, err)
@@ -61,6 +63,7 @@ func TestMemAdmin_Get(t *testing.T) {
 
 	enabled, err = ms.Enabled("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
 
 	err = ms.OnEvent("site1", admin.EvCreate)
 	assert.NoError(t, err)

backend/_example/memory_store/accessor/data.go

@@ -14,9 +14,8 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store/engine"
-
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 const lastLimit = 1000

backend/_example/memory_store/accessor/data_test.go

@@ -425,6 +425,7 @@ func TestMemData_FlagReadOnlyPost(t *testing.T) {
 		Update: engine.FlagTrue}
 	val, err = b.Flag(req)
 	assert.NoError(t, err)
+	assert.True(t, val)
 	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
 	val, err = b.Flag(req)
 	assert.NoError(t, err)
@@ -546,9 +547,9 @@ func TestMemData_FlagListBlocked(t *testing.T) {
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	var blockedIds []string
+	var blockedIds = make([]string, len(blockedList))
-	for _, x := range blockedList {
+	for i, x := range blockedList {
-		blockedIds = append(blockedIds, x.ID)
+		blockedIds[i] = x.ID
 	}
 	require.Equal(t, 2, len(blockedList), b.metaUsers)
 	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.Unlock()
+
+	return nil
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.RUnlock()
+	if !ok {
+		return nil, errors.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.RLock()
+	img, ok := m.imagesStaging[id]
+	m.RUnlock()
+	if !ok {
+		return errors.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.Lock()
+	m.images[id] = img
+	m.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.RUnlock()
+
+	m.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -7,16 +7,22 @@ services:
 
   remark42:
     build:
-      context: .
+      context: ../../..
       dockerfile: Dockerfile
       args:
         - SKIP_BACKEND_TEST=true
         - SKIP_FRONTEND_TEST=true
     image: umputun/remark42:dev
-    container_name: "remark42"
+    container_name: "remark42-dev"
-    hostname: "remark42"
+    hostname: "remark42-dev"
     restart: always
 
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
     ports:
       - "8080:8080" # primary rest server
       - "8084:8084" # local oauth2 server
@@ -33,14 +39,26 @@ services:
       - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
       - STORE_TYPE=rpc
       - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
 
   mem_store.r42:
     image: umputun/mem_store.r42
     build:
-      context: .
+      context: ../../..
       dockerfile: backend/_example/memory_store/Dockerfile
     container_name: "mem_store.r42"
     hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
     environment:
       - API=/cmd
       - SECRET=123456

backend/_example/memory_store/go.mod

@@ -1,16 +1,14 @@
 module github.com/umputun/remark/memory_store
 
-go 1.12
+go 1.14
 
 require (
 	github.com/go-pkgz/jrpc v0.1.0
-	github.com/go-pkgz/lgr v0.6.3
+	github.com/go-pkgz/lgr v0.7.0
-	github.com/jessevdk/go-flags v1.4.0
+	github.com/pkg/errors v0.9.1
-	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.5.1
-	github.com/stretchr/testify v1.4.0
+	github.com/umputun/go-flags v1.5.1
-	github.com/umputun/remark/backend v1.4.0
+	github.com/umputun/remark/backend v1.5.0
 )
 
 replace github.com/umputun/remark/backend => ../../
-
-replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1

backend/_example/memory_store/go.sum

@@ -2,34 +2,34 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
-github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
-github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
-github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.6.0 h1:gcvXlpe0/NoQP3BvneRfgcauLIJDw9VblkoFwZ5XGFs=
+github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
+github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
+github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
-github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
-github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
-github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -39,27 +39,30 @@ github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7z
 github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
 github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
 github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
-github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
-github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
 github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/chi v4.1.0+incompatible h1:ETj3cggsVIY2Xao5ExCu6YhEh5MD6JTfcBzS37R260w=
+github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-pkgz/auth v0.10.1/go.mod h1:wxyQqc0UUP1jT4l6zk1r6XPcVdcgIzW2OiQ8hBEHd64=
-github.com/go-pkgz/auth v0.9.0/go.mod h1:8Kiold3xA9VUiYZwMyMhwBgiGImZFoL/EmtO9lSIQxo=
 github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
 github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
 github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
-github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
-github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
 github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
 github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
 github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
 github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
 github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
 github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
@@ -67,176 +70,141 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.7.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
 github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
-github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
 github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
-github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a h1:Yu34BogBivvmu7SAzHHaB9nZWH5D1C+z3F1jyIaYZSQ=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -245,22 +213,19 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -268,63 +233,37 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
-gopkg.in/oauth2.v3 v3.11.0/go.mod h1:ZdKwOng79bFq17t/ItHGtL8LJ2dW3tmrdJJhHmqS+uc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/_example/memory_store/main.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Umputun. All rights reserved.
+ * Copyright 2020 Umputun. All rights reserved.
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
@@ -12,7 +12,7 @@ import (
 
 	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
 	"github.com/umputun/remark/memory_store/accessor"
 	"github.com/umputun/remark/memory_store/server"
@@ -41,6 +41,7 @@ func main() {
 
 	dataStore := accessor.NewMemData()
 	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
 
 	rpcServer := jrpc.Server{
 		API:        opts.API,
@@ -51,12 +52,13 @@ func main() {
 		Logger:     log.Default(),
 	}
 
-	srv := server.NewRPC(dataStore, adminStore, &rpcServer)
+	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
 
 	admRec := accessor.AdminRec{
-		SiteID: "remark",
+		SiteID:  "remark",
-		IDs:    []string{"dev_user"},
+		Enabled: true,
-		Email:  "admin@example.com",
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
 	}
 	adminStore.Set("remark", admRec)
 

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	key, err := s.adm.Key()
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,306 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/server/rpc.go

@@ -1,13 +1,17 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
 package server
 
 import (
-	"encoding/json"
-
 	"github.com/go-pkgz/jrpc"
 
-	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
@@ -16,11 +20,12 @@ type RPC struct {
 	*jrpc.Server
 	eng engine.Interface
 	adm admin.Store
+	img image.Store
 }
 
 // NewRPC makes RPC instance and register handlers
-func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC {
+func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC {
-	res := &RPC{eng: e, adm: a, Server: r}
+	res := &RPC{eng: e, adm: a, img: i, Server: r}
 	res.addHandlers()
 	return res
 }
@@ -49,186 +54,13 @@ func (s *RPC) addHandlers() {
 		"enabled": s.admEnabledHndl,
 		"event":   s.admEventHndl,
 	})
-}
 
-func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	// image store handlers
-	comment := store.Comment{}
+	s.Group("image", jrpc.HandlersGroup{
-	if err := json.Unmarshal(params, &comment); err != nil {
+		"save_with_id": s.imgSaveWithIDHndl,
-		return jrpc.Response{Error: err.Error()}
+		"load":         s.imgLoadHndl,
-	}
+		"commit":       s.imgCommitHndl,
-	commentID, err := s.eng.Create(comment)
+		"cleanup":      s.imgCleanupHndl,
-	return jrpc.EncodeResponse(id, commentID, err)
+		"info":         s.imgInfoHndl,
-}
+	})
-
-// Find comments
-func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FindRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	resp, err := s.eng.Find(req)
-	return jrpc.EncodeResponse(id, resp, err)
-}
-
-// Get comment
-func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.GetRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	comment, err := s.eng.Get(req)
-	return jrpc.EncodeResponse(id, comment, err)
-}
-
-// Update comment
-func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	comment := store.Comment{}
-	if err := json.Unmarshal(params, &comment); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	err := s.eng.Update(comment)
-	return jrpc.EncodeResponse(id, nil, err)
-}
-
-// counts for site and users
-func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FindRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	count, err := s.eng.Count(req)
-	return jrpc.EncodeResponse(id, count, err)
-}
-
-// info get post meta info
-func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.InfoRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	info, err := s.eng.Info(req)
-	return jrpc.EncodeResponse(id, info, err)
-}
-
-// flagHndl get and sets flag value
-func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FlagRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	status, err := s.eng.Flag(req)
-	return jrpc.EncodeResponse(id, status, err)
-}
-
-// listFlagsHndl list flags for given request
-func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FlagRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	flags, err := s.eng.ListFlags(req)
-	return jrpc.EncodeResponse(id, flags, err)
-}
-
-// userDetailHndl sets or gets single detail value, or gets all details for requested site.
-// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
-// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
-func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.UserDetailRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	value, err := s.eng.UserDetail(req)
-	return jrpc.EncodeResponse(id, value, err)
-}
-
-// deleteHndl delete post(s), user, comment, user details, or everything
-func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.DeleteRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	err := s.eng.Delete(req)
-	return jrpc.EncodeResponse(id, nil, err)
-}
-
-// close store
-func (s *RPC) closeHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
-	if err := s.eng.Close(); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.Response{}
-}
-
-// get admin key
-func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
-	key, err := s.adm.Key()
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, key, err)
-}
-
-// get admins list
-func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	admins, err := s.adm.Admins(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, admins, err)
-}
-
-// get admin email
-func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	email, err := s.adm.Email(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, email, err)
-}
-
-// return site enabled status
-func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	ok, err := s.adm.Enabled(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, ok, err)
-}
-
-// onEvent returns nothing, callback to OnEvent
-func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	ps := []interface{}{}
-	if err := json.Unmarshal(params, &ps); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	siteID, ok := ps[0].(string)
-	if !ok {
-		return jrpc.Response{Error: "wrong siteID type"}
-	}
-	evType, ok := ps[1].(float64)
-	if !ok {
-		return jrpc.Response{Error: "wrong event type"}
-	}
-	err := s.adm.OnEvent(siteID, admin.EventType(evType))
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, nil, err)
 }

backend/_example/memory_store/server/rpc_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Umputun. All rights reserved.
+ * Copyright 2020 Umputun. All rights reserved.
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
@@ -15,370 +15,11 @@ import (
 	"time"
 
 	"github.com/go-pkgz/jrpc"
-	log "github.com/go-pkgz/lgr"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
 
 	"github.com/umputun/remark/memory_store/accessor"
 )
 
-func TestRPC_createHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-}
-
-func TestRPC_findHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	comments, err := re.Find(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 0, len(comments))
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	comments, err = re.Find(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, len(comments))
-	assert.Equal(t, c, comments[0])
-}
-
-func TestRPC_getHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	req := engine.GetRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-
-	_, err := re.Get(req)
-	assert.EqualError(t, err, "not found")
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	comment, err := re.Get(req)
-	assert.NoError(t, err)
-	assert.Equal(t, c, comment)
-}
-
-func TestRPC_updateHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	err := re.Update(c)
-	assert.EqualError(t, err, "not found")
-
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	c.Text = "updates"
-	err = re.Update(c)
-	assert.NoError(t, err)
-
-	req := engine.GetRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-	comment, err := re.Get(req)
-	assert.NoError(t, err)
-	assert.Equal(t, c, comment)
-}
-
-func TestRPC_countHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	count, err := re.Count(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 0, count)
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	count, err = re.Count(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, count)
-}
-
-func TestRPC_infoHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	info, err := re.Info(infoReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, len(info))
-	i := info[0]
-	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
-}
-
-func TestRPC_flagHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	flagReq := engine.FlagRequest{
-		Flag: engine.Verified,
-		Locator: store.Locator{
-			SiteID: "test-site",
-		},
-		UserID: "u1",
-	}
-	status, err := re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, false, status)
-
-	flagReq.Update = engine.FlagTrue
-	status, err = re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-
-	flagReq.Update = engine.FlagNonSet
-	status, err = re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-}
-
-func TestRPC_listFlagsHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	flagReq := engine.FlagRequest{
-		Flag:   engine.Verified,
-		UserID: "u1",
-		Locator: store.Locator{
-			SiteID: "test-site",
-		},
-	}
-	flags, err := re.ListFlags(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, []interface{}{}, flags)
-
-	flagReq.Update = engine.FlagTrue
-	status, err := re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-
-	flags, err = re.ListFlags(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, []interface{}{"u1"}, flags)
-}
-
-func TestRPC_userDetailHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	// add to entries to DB before we start
-	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
-	assert.NoError(t, err, "No error inserting entry expected")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
-	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
-	assert.NoError(t, err, "No error inserting entry expected")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
-
-	// try to change existing entry with wrong SiteID
-	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
-	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
-
-	// stateless tests without changing the state we set up before
-	var testData = []struct {
-		req      engine.UserDetailRequest
-		error    string
-		expected []engine.UserDetailEntry
-	}{
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{}},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{}},
-		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
-			error: `userid cannot be empty in request for single detail`},
-		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
-			error: `unsupported detail "bad"`},
-		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
-			error: `unsupported request with userdetail all`},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
-			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
-	}
-
-	for i, x := range testData {
-		result, err := re.UserDetail(x.req)
-		if x.error != "" {
-			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
-		} else {
-			assert.NoError(t, err, "Error is not expected expected for case %d", i)
-		}
-		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
-	}
-}
-
-func TestRPC_deleteHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	req := engine.DeleteRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-
-	err := re.Delete(req)
-	assert.EqualError(t, err, "not found")
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	err = re.Delete(req)
-	assert.NoError(t, err)
-}
-
-func TestRPC_closeHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := re.Close()
-	assert.NoError(t, err)
-}
-func TestRPC_admKeyHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	key, err := ra.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-}
-
-func TestRPC_admAdminsHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Admins("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	admins, err := ra.Admins("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"id1", "id2"}, admins)
-}
-
-func TestRPC_admEmailHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Admins("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	email, err := ra.Email("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, "admin@example.com", email)
-}
-
-func TestRPC_admEnabledHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Enabled("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	ok, err := ra.Enabled("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, true, ok)
-
-	ok, err = ra.Enabled("test-site-disabled")
-	assert.NoError(t, err)
-	assert.Equal(t, false, ok)
-}
-
-func TestRPC_admEventHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := ra.OnEvent("bad site", admin.EvCreate)
-	assert.EqualError(t, err, "site bad site not found")
-
-	err = ra.OnEvent("test-site", admin.EvCreate)
-	assert.NoError(t, err)
-}
-
 func chooseRandomUnusedPort() (port int) {
 	for i := 0; i < 10; i++ {
 		port = 40000 + int(rand.Int31n(10000))
@@ -402,10 +43,11 @@ func waitForHTTPServerStart(port int) {
 	}
 }
 
-func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
+func prepTestStore(t *testing.T) (port int, teardown func()) {
 	mg := accessor.NewMemData()
 	adm := accessor.NewMemAdminStore("secret")
-	s = NewRPC(mg, adm, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+	img := accessor.NewMemImageStore()
+	s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
 
 	admRec := accessor.AdminRec{
 		SiteID:  "test-site",
@@ -421,12 +63,12 @@ func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
 
 	port = chooseRandomUnusedPort()
 	go func() {
-		log.Printf("%v", s.Run(port))
+		_ = s.Run(port)
 	}()
 
 	waitForHTTPServerStart(port)
 
-	return s, port, func() {
+	return port, func() {
 		require.NoError(t, s.Shutdown())
 	}
 }

backend/app/cmd/avatar.go

@@ -3,9 +3,9 @@ package cmd
 import (
 	"path"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth/avatar"
 )
@@ -34,7 +34,7 @@ func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 }
 
 // Execute runs  with AvatarCommand parameters, entry point for "avatar" command
-func (ac *AvatarCommand) Execute(args []string) error {
+func (ac *AvatarCommand) Execute(_ []string) error {
 	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
 
 	src, err := ac.makeAvatarStore(ac.AvatarSrc)

backend/app/cmd/avatar_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 
 	"github.com/go-pkgz/auth/avatar"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 )
 
 func TestAvatar_Execute(t *testing.T) {
@@ -42,7 +42,7 @@ type avatarMigratorMock struct {
 	retCount int
 }
 
-func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
+func (a *avatarMigratorMock) Migrate(_, _ avatar.Store) (int, error) {
 	a.called++
 	return a.retCount, a.retError
 }

backend/app/cmd/backup.go

@@ -24,7 +24,7 @@ type BackupCommand struct {
 }
 
 // Execute runs export with ExportCommand parameters, entry point for "export" command
-func (ec *BackupCommand) Execute(args []string) error {
+func (ec *BackupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 

backend/app/cmd/backup_test.go

@@ -8,7 +8,8 @@ import (
 	"os"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/cleanup.go

@@ -34,7 +34,7 @@ var (
 
 // Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
 // This command uses provided flags to detect and remove junk comments
-func (cc *CleanupCommand) Execute(args []string) error {
+func (cc *CleanupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] cleanup for site %s", cc.Site)
 
 	posts, err := cc.postsInRange(cc.From, cc.To)

backend/app/cmd/cleanup_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -46,6 +46,7 @@ func TestCleanup_IsSpam(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
 		t.Run(checkName, func(t *testing.T) {
 			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
@@ -144,7 +145,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 }
 
 func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
-	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
 		list := []store.PostInfo{
@@ -165,9 +166,9 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 			},
 		}
 		require.NoError(t, json.NewEncoder(w).Encode(list))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/find", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "remark", r.URL.Query().Get("site"))
 		require.Equal(t, "plain", r.URL.Query().Get("format"))
@@ -193,22 +194,22 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		}
 
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
 }

backend/app/cmd/import.go

@@ -26,7 +26,7 @@ type ImportCommand struct {
 }
 
 // Execute runs import with ImportCommand parameters, entry point for "import" command
-func (ic *ImportCommand) Execute(args []string) error {
+func (ic *ImportCommand) Execute(_ []string) error {
 	log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 
@@ -69,7 +69,7 @@ func (ic *ImportCommand) Execute(args []string) error {
 
 // reader returns reader for file. For .gz file wraps with gunzip
 func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
-	inpFile, err := os.Open(inp)
+	inpFile, err := os.Open(inp) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
 	}

backend/app/cmd/import_test.go

@@ -9,7 +9,8 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/remap.go

@@ -23,7 +23,7 @@ type RemapCommand struct {
 }
 
 // Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
-func (rc *RemapCommand) Execute(args []string) error {
+func (rc *RemapCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 

backend/app/cmd/remap_test.go

@@ -6,7 +6,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/restore_test.go

@@ -7,7 +7,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/server.go

@@ -14,13 +14,13 @@ import (
 	"syscall"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
 	"github.com/kyokomi/emoji"
 	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
@@ -47,7 +47,7 @@ type ServerCommand struct {
 	Cache      CacheGroup      `group:"cache" namespace:"cache" env-namespace:"CACHE"`
 	Admin      AdminGroup      `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify     NotifyGroup     `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
-	SMTP       SmtpGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
+	SMTP       SMTPGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
 	Image      ImageGroup      `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 	Stream     StreamGroup     `group:"stream" namespace:"stream" env-namespace:"STREAM"`
@@ -129,7 +129,7 @@ type StoreGroup struct {
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"rpc" default:"fs"` // nolint
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -138,9 +138,10 @@ type ImageGroup struct {
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	MaxSize      int      `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
-	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
+	ResizeWidth  int      `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
-	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
+	ResizeHeight int      `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
+	RPC          RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // AvatarGroup defines options group for avatar params
@@ -176,8 +177,8 @@ type AdminGroup struct {
 	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
-// SmtpGroup defines options for SMTP server connection, used in auth and notify modules
+// SMTPGroup defines options for SMTP server connection, used in auth and notify modules
-type SmtpGroup struct {
+type SMTPGroup struct {
 	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
 	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
 	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
@@ -199,6 +200,7 @@ type NotifyGroup struct {
 	Email struct {
 		From                string `long:"fromAddress" env:"FROM" description:"from email address"`
 		VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
+		AdminNotifications  bool   `long:"notify_admin" env:"ADMIN" description:"notify admin on new comments via ADMIN_SHARED_EMAIL"`
 	} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 }
 
@@ -244,11 +246,12 @@ type serverApp struct {
 	avatarStore   avatar.Store
 	notifyService *notify.Service
 	imageService  *image.Service
+	authenticator *auth.Service
 	terminated    chan struct{}
 }
 
 // Execute is the entry point for "server" command, called by flag parser
-func (s *ServerCommand) Execute(args []string) error {
+func (s *ServerCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start server on port %d", s.Port)
 	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC", "ADMIN_PASSWD")
 
@@ -335,7 +338,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make pictures store")
 	}
-	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
+	log.Printf("[DEBUG] image service for url=%s, EditDuration=%v", imageService.ImageAPI, imageService.EditDuration)
 
 	dataService := &service.DataStore{
 		Engine:                 storeEngine,
@@ -370,7 +373,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
-		UrlMapperMaker:    migrator.NewUrlMapper,
+		URLMapperMaker:    migrator.NewURLMapper,
 		KeyStore:          adminStore,
 	}
 
@@ -435,6 +438,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		SimpleView:         s.SimpleView,
 	}
 
+	// enable admin notifications only if admin email is set
+	if s.Notify.Email.AdminNotifications && s.Admin.Shared.Email != "" {
+		srv.AdminEmail = s.Admin.Shared.Email
+	}
+
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
@@ -456,6 +464,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
 		imageService:  imageService,
+		authenticator: authenticator,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -471,18 +480,6 @@ func (a *serverApp) run(ctx context.Context) error {
 		<-ctx.Done()
 		log.Print("[INFO] shutdown initiated")
 		a.restSrv.Shutdown()
-		if a.devAuth != nil {
-			a.devAuth.Shutdown()
-		}
-		if e := a.dataService.Close(); e != nil {
-			log.Printf("[WARN] failed to close data store, %s", e)
-		}
-		if e := a.avatarStore.Close(); e != nil {
-			log.Printf("[WARN] failed to close avatar store, %s", e)
-		}
-		a.notifyService.Close()
-		a.imageService.Close()
-		log.Print("[INFO] shutdown completed")
 	}()
 
 	a.activateBackup(ctx) // runs in goroutine for each site
@@ -490,9 +487,31 @@ func (a *serverApp) run(ctx context.Context) error {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
 
+	// staging images resubmit after restart of the app
+	if e := a.dataService.ResubmitStagingImages(a.Sites); e != nil {
+		log.Printf("[WARN] failed to resubmit comments with staging images, %s", e)
+	}
+
 	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
 
 	a.restSrv.Run(a.Port)
+
+	// shutdown procedures after HTTP server is stopped
+	if a.devAuth != nil {
+		a.devAuth.Shutdown()
+	}
+	if e := a.dataService.Close(); e != nil {
+		log.Printf("[WARN] failed to close data store, %s", e)
+	}
+	if e := a.avatarStore.Close(); e != nil {
+		log.Printf("[WARN] failed to close avatar store, %s", e)
+	}
+	a.notifyService.Close()
+	// call potentially infinite loop with cancellation after a minute as a safeguard
+	minuteCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	a.imageService.Close(minuteCtx)
+
 	close(a.terminated)
 	return nil
 }
@@ -565,39 +584,38 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 }
 
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	imageServiceParams := image.ServiceParams{
+		ImageAPI:     s.RemarkURL + "/api/v1/picture/",
+		ProxyAPI:     s.RemarkURL + "/api/v1/img",
+		EditDuration: s.EditDuration,
+		MaxSize:      s.Image.MaxSize,
+		MaxHeight:    s.Image.ResizeHeight,
+		MaxWidth:     s.Image.ResizeWidth,
+	}
 	switch s.Image.Type {
 	case "bolt":
-		boltImageStore, err := image.NewBoltStorage(
+		boltImageStore, err := image.NewBoltStorage(s.Image.Bolt.File, bolt.Options{})
-			s.Image.Bolt.File,
-			s.Image.MaxSize,
-			s.Image.ResizeHeight,
-			s.Image.ResizeWidth,
-			bolt.Options{},
-		)
 		if err != nil {
 			return nil, err
 		}
-		return &image.Service{
+		return image.NewService(boltImageStore, imageServiceParams), nil
-			Store:    boltImageStore,
-			ImageAPI: s.RemarkURL + "/api/v1/picture/",
-			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
-		}, nil
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
 		}
-		return &image.Service{
+		return image.NewService(&image.FileSystem{
-			Store: &image.FileSystem{
+			Location:   s.Image.FS.Path,
-				Location:   s.Image.FS.Path,
+			Staging:    s.Image.FS.Staging,
-				Staging:    s.Image.FS.Staging,
+			Partitions: s.Image.FS.Partitions,
-				Partitions: s.Image.FS.Partitions,
+		}, imageServiceParams), nil
-				MaxSize:    s.Image.MaxSize,
+	case "rpc":
-				MaxHeight:  s.Image.ResizeHeight,
+		return image.NewService(&image.RPC{
-				MaxWidth:   s.Image.ResizeWidth,
+			Client: jrpc.Client{
-			},
+				API:        s.Image.RPC.API,
-			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+				Client:     http.Client{Timeout: s.Image.RPC.TimeOut},
-			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
+				AuthUser:   s.Image.RPC.AuthUser,
-		}, nil
+				AuthPasswd: s.Image.RPC.AuthPassword,
+			}}, imageServiceParams), nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
 }
@@ -767,7 +785,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
 				VerificationSubject: s.Notify.Email.VerificationSubject,
 				UnsubscribeURL:      s.RemarkURL + "/email/unsubscribe.html",
 				// TODO: uncomment after #560 frontend part is ready and URL is known
-				//SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
+				// SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
 				TokenGenFn: func(userID, email, site string) (string, error) {
 					claims := token.Claims{
 						Handshake: &token.Handshake{ID: userID + "::" + email},
@@ -785,7 +803,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
 					return tkn, nil
 				},
 			}
-			smtpParams := notify.SmtpParams{
+			smtpParams := notify.SMTPParams{
 				Host:     s.SMTP.Host,
 				Port:     s.SMTP.Port,
 				TLS:      s.SMTP.TLS,
@@ -849,7 +867,7 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 		TokenDuration:  s.Auth.TTL.JWT,
 		CookieDuration: s.Auth.TTL.Cookie,
 		SecureCookies:  strings.HasPrefix(s.RemarkURL, "https://"),
-		SecretReader: token.SecretFunc(func() (string, error) { // get secret per site
+		SecretReader: token.SecretFunc(func(aud string) (string, error) { // get secret per site
 			return admns.Key()
 		}),
 		ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
@@ -863,6 +881,21 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			if err != nil {
 				log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
 			}
+
+			// don't allow anonymous with admin's name
+			if strings.HasPrefix(c.User.ID, "anonymous_") {
+				admins, err := admns.Admins(c.Audience)
+				if err != nil {
+					log.Printf("[WARN] can't get admins for %s, %v", c.Audience, err)
+				}
+				for _, a := range admins {
+					if strings.EqualFold(c.User.Name, a) {
+						c.User.SetBoolAttr("blocked", true)
+						break
+					}
+				}
+			}
+
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,

backend/app/cmd/server_test.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -114,27 +114,74 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
+	// try to add a comment as good anonymous
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims := getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+
 	// try to login with bad name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
+	// try to login with admin name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umputun&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as anonymous with admin name
+	client = http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims = getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	assert.True(t, claims.User.BoolAttr("blocked"), "should be blocked")
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+
 	cancel()
 	app.Wait()
 }
 
+func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (token string, claims token.Claims) {
+	var err error
+	for _, c := range resp.Cookies() {
+		if c.Name == "JWT" {
+			token = c.Value
+			claims, err = app.restSrv.Authenticator.TokenService().Parse(c.Value)
+			require.NoError(t, err)
+		}
+	}
+	return token, claims
+}
+
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
 	sslPort := chooseRandomUnusedPort()
@@ -561,6 +608,8 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
 	cmd.SMTP.Password = "test_password"
 	cmd.SMTP.TimeOut = time.Second
 	cmd.UpdateLimit = 10
+	cmd.Admin.Type = "shared"
+	cmd.Admin.Shared.Admins = []string{"umputun", "bobuk"}
 	cmd = fn(cmd)
 
 	os.Remove(cmd.Store.Bolt.Path + "/remark.db")

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )
@@ -84,6 +84,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
+// nolint:gochecknoinits
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal)

backend/app/main_test.go

@@ -13,7 +13,6 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -31,8 +30,8 @@ func Test_Main(t *testing.T) {
 	done := make(chan struct{})
 	go func() {
 		<-done
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.NoError(t, err)
+		require.NoError(t, e)
 	}()
 
 	finished := make(chan struct{})
@@ -62,7 +61,7 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "goroutine"))
 	assert.True(t, strings.Contains(dump, "[running]"))
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
-	log.Printf("\n dump: %s", dump)
+	t.Logf("\n dump: %s", dump)
 }
 
 func chooseRandomUnusedPort() (port int) {

backend/app/migrator/backup_test.go

@@ -77,7 +77,7 @@ func TestBackup_Do(t *testing.T) {
 
 type mockExporter struct{}
 
-func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
+func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
 	_, err := w.Write([]byte("some export blah blah 1234567890"))
 	return 1000, err
 }

backend/app/migrator/disqus_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"

backend/app/migrator/mapper.go

@@ -7,15 +7,15 @@ import (
 	"strings"
 )
 
-// UrlMapper implements Mapper interface
+// URLMapper implements Mapper interface
-type UrlMapper struct {
+type URLMapper struct {
 	rules map[string]string
 }
 
-// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
+// NewURLMapper reads rules from given reader and returns initialized URLMapper
 // if given rules are valid.
-func NewUrlMapper(reader io.Reader) (Mapper, error) {
+func NewURLMapper(reader io.Reader) (Mapper, error) {
-	u := &UrlMapper{}
+	u := &URLMapper{}
 	if err := u.loadRules(reader); err != nil {
 		return u, err
 	}
@@ -29,7 +29,7 @@ func NewUrlMapper(reader io.Reader) (Mapper, error) {
 // Example:
 // https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
 // https://www.myblog.com/* https://myblog.com/*
-func (u *UrlMapper) loadRules(reader io.Reader) error {
+func (u *URLMapper) loadRules(reader io.Reader) error {
 	data, err := ioutil.ReadAll(reader)
 	if err != nil {
 		return err
@@ -54,19 +54,19 @@ func (u *UrlMapper) loadRules(reader io.Reader) error {
 
 // URL maps given url to another url according loaded url-rules.
 // If not matched returns given url.
-func (u *UrlMapper) URL(url string) string {
+func (u *URLMapper) URL(url string) string {
-	if newUrl, ok := u.rules[url]; ok {
+	if newURL, ok := u.rules[url]; ok {
-		return newUrl
+		return newURL
 	}
 	// try to match by prefix
-	for oldUrl, newUrl := range u.rules {
+	for oldURL, newURL := range u.rules {
-		if !strings.HasSuffix(oldUrl, "*") {
+		if !strings.HasSuffix(oldURL, "*") {
 			continue
 		}
-		oldUrl = strings.TrimSuffix(oldUrl, "*")
+		oldURL = strings.TrimSuffix(oldURL, "*")
-		newUrl = strings.TrimSuffix(newUrl, "*")
+		newURL = strings.TrimSuffix(newURL, "*")
-		if strings.HasPrefix(url, oldUrl) {
+		if strings.HasPrefix(url, oldURL) {
-			return newUrl + strings.TrimPrefix(url, oldUrl)
+			return newURL + strings.TrimPrefix(url, oldURL)
 		}
 	}
 	// search failed, return given url

backend/app/migrator/mapper_test.go

@@ -16,7 +16,7 @@ https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/
 https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
 `)
 
-	mapper, err := NewUrlMapper(rules)
+	mapper, err := NewURLMapper(rules)
 	assert.NoError(t, err)
 
 	// if url not matched mapper should return given url
@@ -30,7 +30,7 @@ https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.c
 
 	// want remap from http to https
 	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
-	mapper, err = NewUrlMapper(rules)
+	mapper, err = NewURLMapper(rules)
 	assert.NoError(t, err)
 	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
 	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
@@ -38,7 +38,7 @@ https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.c
 
 	// want remap from http to https by pattern
 	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
-	mapper, err = NewUrlMapper(rules)
+	mapper, err = NewURLMapper(rules)
 	assert.NoError(t, err)
 	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
 	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
@@ -80,7 +80,7 @@ func TestUrlMapper_New(t *testing.T) {
 		},
 	}
 	for _, c := range cases {
-		_, err := NewUrlMapper(strings.NewReader(c.rules))
+		_, err := NewURLMapper(strings.NewReader(c.rules))
 		if c.expectError {
 			assert.Error(t, err)
 		} else {

backend/app/migrator/migrator_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"

backend/app/migrator/native_test.go

@@ -10,10 +10,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -35,7 +34,7 @@ func TestNative_Export(t *testing.T) {
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
-	log.Print(c1)
+	t.Log(c1)
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
@@ -106,7 +105,7 @@ func TestNative_ImportWithMapper(t *testing.T) {
 
 	// want to remap comments to https://rdt.c
 	rules := `https://radio-t.com* https://rdt.c*`
-	mapper, err := NewUrlMapper(strings.NewReader(rules))
+	mapper, err := NewURLMapper(strings.NewReader(rules))
 	assert.NoError(t, err)
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}

backend/app/migrator/wordpress_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -190,7 +190,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/notify/email.go

@@ -29,8 +29,8 @@ type EmailParams struct {
 	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
 }
 
-// SmtpParams contain settings for smtp server connection
+// SMTPParams contain settings for smtp server connection
-type SmtpParams struct {
+type SMTPParams struct {
 	Host     string        // SMTP host
 	Port     int           // SMTP port
 	TLS      bool          // TLS auth
@@ -42,7 +42,7 @@ type SmtpParams struct {
 // Email implements notify.Destination for email
 type Email struct {
 	EmailParams
-	SmtpParams
+	SMTPParams
 
 	smtp       smtpClientCreator
 	msgTmpl    *template.Template // parsed request message template
@@ -64,7 +64,7 @@ type smtpClient interface {
 
 // smtpClientCreator interface defines function for creating new smtpClients
 type smtpClientCreator interface {
-	Create(SmtpParams) (smtpClient, error)
+	Create(SMTPParams) (smtpClient, error)
 }
 
 type emailMessage struct {
@@ -88,6 +88,7 @@ type msgTmplData struct {
 	PostTitle         string
 	Email             string
 	UnsubscribeLink   string
+	ForAdmin          bool
 }
 
 // verifyTmplData store data for verification message template execution
@@ -122,48 +123,52 @@ const (
 		p {
 			margin: 0 0 12px;
 		}
+		blockquote {
+			margin: 10px 0;
+			padding: 12px 12px 1px 12px;
+			background: rgba(255,255,255,.5)
+		}
 	</style>
 </head>
 <!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
 <body>
 	<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
 		<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
+        {{- if .ForAdmin}}
+		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+        {{- else }}
 		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+        {{- end }}
 		<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
-			<div>
+            {{- if .ParentCommentText}}
-				<div style="margin-bottom: 12px; line-height: 24px;">
+			<div style="margin-bottom: 12px; line-height: 24px; word-break: break-all;">
-					<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+				<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; display: inline; vertical-align: middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
-					<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
+				<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
-					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
+				<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
-					<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
+				<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
-				</div>
-				<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
-					{{.ParentCommentText}}
-				</div>
 			</div>
-			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px; line-height: 24px;">
+			<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
-				<div style="margin-bottom: 8px;">
+				{{.ParentCommentText}}
-					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+			</div>
-					<div style="float: left; font-size: 14px; font-weight: bold; color: #777">
+            {{- end }}
-						{{.UserName}}
+			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px;">
-					</div>
+				<div style="margin-bottom: 12px;" line-height: 24px;word-break: break-all;>
-					<div style="color: #999; font-size: 14px; margin: 0 8px; float: left;">
+					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; display:inline; vertical-align:middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
-						{{.CommentDate.Format "02.01.2006 at 15:04"}}
+					<span style="font-size: 14px; font-weight: bold; color: #777">{{.UserName}}</span>
-					</div>
+					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.CommentDate.Format "02.01.2006 at 15:04"}}</span>
 					<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
 				</div>
-				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">
+				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">{{.CommentText}}</div>
-					{{.CommentText}}
-				</div>
 			</div>
 		</div>
 		<div style="text-align: center; font-size: 14px; margin-top: 32px;">
-			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a> for {{.ParentUserName}}</i>
+			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a>{{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}</i>
-			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;">
+			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;"></div>
-				<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
+			{{- if .UnsubscribeLink}}
-				<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
+			<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
-				<div style="opacity: 0;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
+			{{- end }}
-			</div>
+			<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
+			<div style="opacity: 0;font-size: 1;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
 		</div>
 	</div>
 </body>
@@ -180,10 +185,10 @@ const (
 	<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
 		<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
 		<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
-		{{if .SubscribeURL}}
+		{{- if .SubscribeURL}}
 		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
 		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
-		{{ end }}
+		{{- end }}
 		<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
 			<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
 			<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
@@ -197,7 +202,7 @@ const (
 )
 
 // NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
-func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
+func NewEmail(emailParams EmailParams, smtpParams SMTPParams) (*Email, error) {
 	// set up Email emailParams
 	res := Email{EmailParams: emailParams}
 	if res.MsgTemplate == "" {
@@ -212,7 +217,7 @@ func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
 
 	// set up SMTP emailParams
 	res.smtp = &emailClient{}
-	res.SmtpParams = smtpParams
+	res.SMTPParams = smtpParams
 	if res.TimeOut <= 0 {
 		res.TimeOut = defaultEmailTimeout
 	}
@@ -220,7 +225,7 @@ func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
 	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
 		res.Host, res.Username, res.TimeOut)
 
-	// initialise templates
+	// initialize templates
 	var err error
 	if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
 		return nil, errors.Wrapf(err, "can't parse message template")
@@ -231,10 +236,9 @@ func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
 	return &res, err
 }
 
-// Send email about reply to Request.Email if it's set, otherwise do nothing and return nil, thread safe
+// Send email about comment reply to Request.Email if it's set,
-// do not returns sending error, only following:
+// also sends email to site administrator if appropriate option is set.
-// 1. (likely impossible) template execution error from email message creation from Request
+// Thread safe
-// 2. message dropped without sending in case of closed ctx
 func (e *Email) Send(ctx context.Context, req Request) (err error) {
 	if req.Email == "" {
 		// this means we can't send this request via Email
@@ -256,12 +260,12 @@ func (e *Email) Send(ctx context.Context, req Request) (err error) {
 	}
 
 	if req.Comment.ID != "" {
-		if req.parent.User.ID == req.Comment.User.ID {
+		if req.parent.User.ID == req.Comment.User.ID && !req.ForAdmin {
 			// don't send anything if if user replied to their own comment
 			return nil
 		}
 		log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
-		msg, err = e.buildMessageFromRequest(req)
+		msg, err = e.buildMessageFromRequest(req, req.ForAdmin)
 		if err != nil {
 			return err
 		}
@@ -292,33 +296,46 @@ func (e *Email) buildVerificationMessage(user, email, token, site string) (strin
 }
 
 // buildMessageFromRequest generates email message based on Request using e.MsgTemplate
-func (e *Email) buildMessageFromRequest(req Request) (string, error) {
+func (e *Email) buildMessageFromRequest(req Request, forAdmin bool) (string, error) {
 	subject := "New reply to your comment"
+	if forAdmin {
+		subject = "New comment to your site"
+	}
 	if req.Comment.PostTitle != "" {
 		subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
 	}
+
 	token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
-	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
 	if err != nil {
 		return "", errors.Wrapf(err, "error creating token for unsubscribe link")
 	}
-	commentUrlPrefix := req.Comment.Locator.URL + uiNav
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if forAdmin {
+		unsubscribeLink = ""
+	}
+
+	commentURLPrefix := req.Comment.Locator.URL + uiNav
 	msg := bytes.Buffer{}
-	err = e.msgTmpl.Execute(&msg, msgTmplData{
+	tmplData := msgTmplData{
-		UserName:          req.Comment.User.Name,
+		UserName:        req.Comment.User.Name,
-		UserPicture:       req.Comment.User.Picture,
+		UserPicture:     req.Comment.User.Picture,
-		CommentText:       req.Comment.Text,
+		CommentText:     req.Comment.Text,
-		CommentLink:       commentUrlPrefix + req.Comment.ID,
+		CommentLink:     commentURLPrefix + req.Comment.ID,
-		CommentDate:       req.Comment.Timestamp,
+		CommentDate:     req.Comment.Timestamp,
-		ParentUserName:    req.parent.User.Name,
+		PostTitle:       req.Comment.PostTitle,
-		ParentUserPicture: req.parent.User.Picture,
+		Email:           req.Email,
-		ParentCommentText: req.parent.Text,
+		UnsubscribeLink: unsubscribeLink,
-		ParentCommentLink: commentUrlPrefix + req.parent.ID,
+		ForAdmin:        forAdmin,
-		ParentCommentDate: req.parent.Timestamp,
+	}
-		PostTitle:         req.Comment.PostTitle,
+	// in case of message to admin, parent message might be empty
-		Email:             req.Email,
+	if req.Comment.ParentID != "" {
-		UnsubscribeLink:   unsubscribeLink,
+		tmplData.ParentUserName = req.parent.User.Name
-	})
+		tmplData.ParentUserPicture = req.parent.User.Picture
+		tmplData.ParentCommentText = req.parent.Text
+		tmplData.ParentCommentLink = commentURLPrefix + req.parent.ID
+		tmplData.ParentCommentDate = req.parent.Timestamp
+	}
+	err = e.msgTmpl.Execute(&msg, tmplData)
 	if err != nil {
 		return "", errors.Wrapf(err, "error executing template to build comment reply message")
 	}
@@ -354,7 +371,10 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
 	if _, err := qp.Write([]byte(body)); err != nil {
 		return "", err
 	}
-	defer qp.Close()
+	// flush now, must NOT use defer, for small body, defer may cause buff.String() got empty body
+	if err := qp.Close(); err != nil {
+		return "", fmt.Errorf("quotedprintable Write failed: %w", err)
+	}
 	m := buff.String()
 	message += "\n" + m
 	return message, nil
@@ -364,30 +384,30 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
 // Thread safe.
 func (e *Email) sendMessage(m emailMessage) error {
 	if e.smtp == nil {
-		return errors.New("sendMessage called without smtpClient set")
+		return errors.New("sendMessage called without client set")
 	}
-	smtpClient, err := e.smtp.Create(e.SmtpParams)
+	client, err := e.smtp.Create(e.SMTPParams)
 	if err != nil {
 		return errors.Wrap(err, "failed to make smtp Create")
 	}
 
 	defer func() {
-		if err := smtpClient.Quit(); err != nil {
+		if err = client.Quit(); err != nil {
 			log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
-			if err := smtpClient.Close(); err != nil {
+			if err = client.Close(); err != nil {
 				log.Printf("[WARN] can't close smtp connection, %v", err)
 			}
 		}
 	}()
 
-	if err := smtpClient.Mail(m.from); err != nil {
+	if err = client.Mail(m.from); err != nil {
 		return errors.Wrapf(err, "bad from address %q", m.from)
 	}
-	if err := smtpClient.Rcpt(m.to); err != nil {
+	if err = client.Rcpt(m.to); err != nil {
 		return errors.Wrapf(err, "bad to address %q", m.to)
 	}
 
-	writer, err := smtpClient.Data()
+	writer, err := client.Data()
 	if err != nil {
 		return errors.Wrap(err, "can't make email writer")
 	}
@@ -411,9 +431,9 @@ func (e *Email) String() string {
 	return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
 }
 
-// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SmtpParams
+// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SMTPParams
 // and returns pointer to it. Thread safe.
-func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
+func (s *emailClient) Create(params SMTPParams) (smtpClient, error) {
 	authenticate := func(c *smtp.Client) error {
 		if params.Username == "" || params.Password == "" {
 			return nil
@@ -447,7 +467,7 @@ func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
 		return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
 	}
 
-	c, err = smtp.NewClient(conn, srvAddress)
+	c, err = smtp.NewClient(conn, params.Host)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to dial")
 	}

backend/app/notify/email_test.go

@@ -22,7 +22,7 @@ func TestEmailNew(t *testing.T) {
 		err         bool
 		errText     string
 		emailParams EmailParams
-		smtpParams  SmtpParams
+		smtpParams  SMTPParams
 	}{
 		{name: "empty"},
 		{name: "with template parse error",
@@ -36,7 +36,7 @@ func TestEmailNew(t *testing.T) {
 				From:                 "test@from",
 				VerificationTemplate: "{{",
 			},
-			smtpParams: SmtpParams{
+			smtpParams: SMTPParams{
 				Host:     "test@host",
 				Port:     1000,
 				TLS:      true,
@@ -50,7 +50,7 @@ func TestEmailNew(t *testing.T) {
 			emailParams: EmailParams{
 				From: "test@from",
 			},
-			smtpParams: SmtpParams{
+			smtpParams: SMTPParams{
 				Host:     "test@host",
 				Port:     1000,
 				TLS:      true,
@@ -61,6 +61,7 @@ func TestEmailNew(t *testing.T) {
 		},
 	}
 	for _, d := range testSet {
+		d := d
 		t.Run(d.name, func(t *testing.T) {
 			email, err := NewEmail(d.emailParams, d.smtpParams)
 
@@ -125,7 +126,7 @@ func TestEmailSendErrors(t *testing.T) {
 }
 
 func TestEmailSend_ExitConditions(t *testing.T) {
-	email, err := NewEmail(EmailParams{}, SmtpParams{})
+	email, err := NewEmail(EmailParams{}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email, "expecting email returned")
 	// prevent triggering e.autoFlush creation
@@ -152,6 +153,7 @@ func TestEmailSendClientError(t *testing.T) {
 			err: "can't make email writer: failed to send"},
 	}
 	for _, d := range testSet {
+		d := d
 		t.Run(d.name, func(t *testing.T) {
 			e := Email{smtp: d.smtp}
 			if d.err != "" {
@@ -179,24 +181,24 @@ func TestEmailSendClientError(t *testing.T) {
 }
 
 func TestEmail_Send(t *testing.T) {
-	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email)
-	fakeSmtp := fakeTestSMTP{}
+	fakeSMTP := fakeTestSMTP{}
-	email.smtp = &fakeSmtp
+	email.smtp = &fakeSMTP
 	email.TokenGenFn = TokenGenFn
 	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
 	req := Request{
-		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
 		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
 		Email:   "test@example.org",
 	}
 	assert.NoError(t, email.Send(context.TODO(), req))
-	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
-	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
-	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
 	// test buildMessageFromRequest separately for message text
-	res, err := email.buildMessageFromRequest(req)
+	res, err := email.buildMessageFromRequest(req, req.ForAdmin)
 	assert.NoError(t, err)
 	assert.Contains(t, res, `From: from@example.org
 To: test@example.org
@@ -206,15 +208,32 @@ MIME-version: 1.0
 Content-Type: text/html; charset="UTF-8"
 List-Unsubscribe-Post: List-Unsubscribe=One-Click
 List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
+Date: `)
+
+	// send email to admin without parent set
+	req = Request{
+		Comment:  store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Email:    "admin@example.org",
+		ForAdmin: true,
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	res, err = email.buildMessageFromRequest(req, req.ForAdmin)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: admin@example.org
+Subject: New comment to your site for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
 Date: `)
 }
 
 func TestEmail_SendVerification(t *testing.T) {
-	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email)
-	fakeSmtp := fakeTestSMTP{}
+	fakeSMTP := fakeTestSMTP{}
-	email.smtp = &fakeSmtp
+	email.smtp = &fakeSMTP
 	email.TokenGenFn = TokenGenFn
 	req := Request{
 		Email: "test@example.org",
@@ -225,9 +244,9 @@ func TestEmail_SendVerification(t *testing.T) {
 		},
 	}
 	assert.NoError(t, email.Send(context.TODO(), req))
-	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
-	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
-	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
 	// test buildMessageFromRequest separately for message text
 	res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
 	assert.NoError(t, err)
@@ -255,7 +274,7 @@ Date: `)
 
 func Test_emailClient_Create(t *testing.T) {
 	creator := emailClient{}
-	client, err := creator.Create(SmtpParams{})
+	client, err := creator.Create(SMTPParams{})
 	assert.Error(t, err, "absence of address to connect results in error")
 	assert.Nil(t, client, "no client returned in case of error")
 }
@@ -271,7 +290,7 @@ type fakeTestSMTP struct {
 	lock       sync.RWMutex
 }
 
-func (f *fakeTestSMTP) Create(SmtpParams) (smtpClient, error) {
+func (f *fakeTestSMTP) Create(SMTPParams) (smtpClient, error) {
 	if f.fail["create"] {
 		return nil, errors.New("failed to create client")
 	}

backend/app/notify/notify.go

@@ -37,9 +37,11 @@ type Store interface {
 
 // Request notification either about comment or about particular user verification
 type Request struct {
-	Comment      store.Comment        // if set sent notifications about new comment
+	Comment  store.Comment // if set sent notifications about new comment
-	parent       store.Comment        // fetched only in case Comment is set
+	parent   store.Comment // fetched only in case Comment is set
-	Email        string               // if set (also) send email
+	Email    string        // if set (also) send email
+	ForAdmin bool          // if set, message supposed to be sent to administrator
+
 	Verification VerificationMetadata // if set sent verification notification
 }
 
@@ -82,9 +84,13 @@ func (s *Service) Submit(req Request) {
 	if s.dataService != nil && req.Comment.ParentID != "" {
 		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
 			req.parent = p
-			req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
+			// user notification, should fetch email for it.
-			if err != nil {
+			// administrator notification comes with pre-set email
-				log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
+			if req.Email == "" {
+				req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
+				if err != nil {
+					log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
+				}
 			}
 		}
 	}

backend/app/notify/notify_mock.go

@@ -9,6 +9,7 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// MockDest is a destination mock
 type MockDest struct {
 	data   []Request
 	id     int
@@ -16,6 +17,7 @@ type MockDest struct {
 	lock   sync.Mutex
 }
 
+// Send mock
 func (m *MockDest) Send(ctx context.Context, r Request) error {
 	m.lock.Lock()
 	defer m.lock.Unlock()
@@ -30,6 +32,7 @@ func (m *MockDest) Send(ctx context.Context, r Request) error {
 	return nil
 }
 
+// Get mock
 func (m *MockDest) Get() []Request {
 	m.lock.Lock()
 	defer m.lock.Unlock()

backend/app/notify/telegram.go

@@ -90,6 +90,11 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
 		// verification request received, send nothing
 		return nil
 	}
+	if req.ForAdmin {
+		// request for administrator received, do nothing with it
+		// as we already sent message on request without this flag set
+		return nil
+	}
 	client := http.Client{Timeout: telegramTimeOut}
 	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
 

backend/app/notify/telegram_test.go

@@ -37,7 +37,9 @@ func TestTelegram_New(t *testing.T) {
 	assert.EqualError(t, err, "unexpected telegram status code 404")
 
 	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
-	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "can't initialize telegram notifications")
+	assert.Contains(t, err.Error(), "dial tcp 127.0.0.1:4321: connect: connection refused")
 
 	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
 	assert.Error(t, err, "empty api url not allowed")

backend/app/rest/api/admin.go

@@ -111,7 +111,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
+	if err = a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
 		code := parseError(err, rest.ErrInternal)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
 		return

backend/app/rest/api/admin_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
@@ -55,8 +55,8 @@ func TestAdmin_Delete(t *testing.T) {
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 
 	// delete a comment
 	req, err := http.NewRequest(http.MethodDelete,
@@ -103,8 +103,8 @@ func TestAdmin_Delete(t *testing.T) {
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 }
 
 func TestAdmin_Title(t *testing.T) {
@@ -310,7 +310,7 @@ func TestAdmin_Block(t *testing.T) {
 	pi = []store.PostInfo{}
 	err = json.Unmarshal(body, &pi)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}), pi)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}, pi)
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)

backend/app/rest/api/migrator.go

@@ -28,7 +28,7 @@ type Migrator struct {
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
 	NativeExporter    migrator.Exporter
-	UrlMapperMaker    migrator.MapperMaker
+	URLMapperMaker    migrator.MapperMaker
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -161,7 +161,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	// create new url-mapper from given rules in body
-	mapper, err := m.UrlMapperMaker(r.Body)
+	mapper, err := m.URLMapperMaker(r.Body)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
 		return
@@ -180,12 +180,12 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		defer func() {
-			if e := os.Remove(fh.Name()); e != nil {
+			if e = os.Remove(fh.Name()); e != nil {
 				log.Printf("[WARN] failed to remove temp file %+v", e)
 			}
 		}()
 		log.Printf("[DEBUG] start export for site=%s", siteID)
-		if _, e := m.NativeExporter.Export(fh, siteID); e != nil {
+		if _, e = m.NativeExporter.Export(fh, siteID); e != nil {
 			log.Printf("[WARN] export failed with %+v", e)
 			return
 		}
@@ -233,7 +233,7 @@ func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	}
 	log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, provider)
 
-	fh, err := os.Open(tmpfile)
+	fh, err := os.Open(tmpfile) // nolint
 	if err != nil {
 		log.Printf("[WARN] import failed, %v", err)
 		return

backend/app/rest/api/migrator_test.go

@@ -168,11 +168,12 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 "picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
 "ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
 "votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
-	recs := []string{}
+	nRecs := 50
-	for i := 0; i < 50; i++ {
+	recs := make([]string, 0, nRecs)
+	for i := 0; i < nRecs; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
-	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with `nRecs` records
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
@@ -183,7 +184,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 5 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=10ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=5ms", nil)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
@@ -457,7 +458,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/rest/api/rest.go

@@ -50,6 +50,7 @@ type Rest struct {
 	AnonVote        bool
 	WebRoot         string
 	RemarkURL       string
+	AdminEmail      string
 	ReadOnlyAge     int
 	SharedSecret    string
 	ScoreThresholds struct {
@@ -364,6 +365,7 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 		authenticator:    s.Authenticator,
 		notifyService:    s.NotifyService,
 		remarkURL:        s.RemarkURL,
+		adminEmail:       s.AdminEmail,
 		anonVote:         s.AnonVote,
 	}
 
@@ -425,7 +427,7 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		CriticalScore:      s.ScoreThresholds.Critical,
 		PositiveScore:      s.DataService.PositiveScore,
 		ReadOnlyAge:        s.ReadOnlyAge,
-		MaxImageSize:       s.ImageService.Store.SizeLimit(),
+		MaxImageSize:       s.ImageService.MaxSize,
 		EmailNotifications: s.EmailNotifications,
 		EmojiEnabled:       s.EmojiEnabled,
 		AnonVote:           s.AnonVote,
@@ -462,7 +464,7 @@ func addFileServer(r chi.Router, path string, root http.FileSystem, version stri
 	origPath := path
 	webFS = http.StripPrefix(path, webFS)
 	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
+		r.Get(path, http.RedirectHandler(path+"/", http.StatusMovedPermanently).ServeHTTP)
 		path += "/"
 	}
 	path += "*"

backend/app/rest/api/rest_private.go

@@ -39,6 +39,7 @@ type private struct {
 	notifyService    *notify.Service
 	authenticator    *auth.Service
 	remarkURL        string
+	adminEmail       string
 	anonVote         bool
 }
 
@@ -58,7 +59,7 @@ type privStore interface {
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 }
 
-const unsubscribeHtml = `<!DOCTYPE html>
+const unsubscribeHTML = `<!DOCTYPE html>
 <html>
 <head>
     <meta name="viewport" content="width=device-width"/>
@@ -131,9 +132,14 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
+	// user notification
 	if s.notifyService != nil {
 		s.notifyService.Submit(notify.Request{Comment: finalComment})
 	}
+	// admin notification
+	if s.notifyService != nil && s.adminEmail != "" {
+		s.notifyService.Submit(notify.Request{Comment: finalComment, Email: s.adminEmail, ForAdmin: true})
+	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
 
@@ -412,7 +418,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] unsubscribe user %s", userID)
 
-	if err := s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
+	if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
 		code := parseError(err, rest.ErrInternal)
 		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
 		return
@@ -437,7 +443,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHtml))
+	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHTML))
 	msg := bytes.Buffer{}
 	MustExecute(tmpl, &msg, nil)
 	render.HTML(w, r, msg.String())
@@ -568,14 +574,14 @@ func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	file, header, err := r.FormFile("file")
+	file, _, err := r.FormFile("file")
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
-	id, err := s.imageService.Save(header.Filename, user.ID, file)
+	id, err := s.imageService.Save(user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return

backend/app/rest/api/rest_private_test.go

@@ -249,8 +249,8 @@ func TestRest_UpdateDelete(t *testing.T) {
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 
 	// delete a comment
 	client := http.Client{}
@@ -290,8 +290,8 @@ func TestRest_UpdateDelete(t *testing.T) {
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	require.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 }
 
 func TestRest_UpdateNotOwner(t *testing.T) {
@@ -548,6 +548,7 @@ func TestRest_Email(t *testing.T) {
 	}
 	client := http.Client{}
 	for _, x := range testData {
+		x := x
 		t.Run(x.description, func(t *testing.T) {
 			req, err := http.NewRequest(x.method, ts.URL+x.url, nil)
 			require.NoError(t, err)
@@ -596,11 +597,12 @@ func TestRest_EmailNotification(t *testing.T) {
 	parentComment := store.Comment{}
 	require.NoError(t, render.DecodeJSON(strings.NewReader(string(body)), &parentComment))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
-	require.Equal(t, 1, len(mockDestination.Get()))
+	require.Equal(t, 2, len(mockDestination.Get()))
-	assert.Equal(t, "", mockDestination.Get()[0].Email)
+	assert.Empty(t, mockDestination.Get()[0].Email)
+	assert.Equal(t, "admin@example.org", mockDestination.Get()[1].Email)
 
-	// create child comment from another user, no email notification expected
+	// create child comment from another user, email notification only to admin expected
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
 		`{"text": "test 456",
 	"pid": "%s",
@@ -615,9 +617,10 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
-	require.Equal(t, 2, len(mockDestination.Get()))
+	require.Equal(t, 4, len(mockDestination.Get()))
-	assert.Empty(t, mockDestination.Get()[1].Email)
+	assert.Empty(t, mockDestination.Get()[2].Email)
+	assert.Equal(t, "admin@example.org", mockDestination.Get()[3].Email)
 
 	// send confirmation token for email
 	req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", nil)
@@ -629,10 +632,10 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
-	require.Equal(t, 3, len(mockDestination.Get()))
+	require.Equal(t, 5, len(mockDestination.Get()))
-	require.NotEmpty(t, mockDestination.Get()[2].Verification)
+	require.NotEmpty(t, mockDestination.Get()[4].Verification)
-	verificationToken := mockDestination.Get()[2].Verification.Token
+	verificationToken := mockDestination.Get()[4].Verification.Token
 
 	// verify email
 	req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), nil)
@@ -674,9 +677,9 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
-	require.Equal(t, 4, len(mockDestination.Get()))
+	require.Equal(t, 7, len(mockDestination.Get()))
-	assert.Equal(t, "good@example.com", mockDestination.Get()[3].Email)
+	assert.Equal(t, "good@example.com", mockDestination.Get()[5].Email)
 
 	// delete user's email
 	req, err = http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/email?site=remark42", nil)
@@ -688,7 +691,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 
-	// create child comment from another user, no email notification expected
+	// create child comment from another user, no email notification expected except for admin
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 321",
 	"user": {"name": "other_user"},
@@ -702,9 +705,9 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
-	require.Equal(t, 5, len(mockDestination.Get()))
+	require.Equal(t, 9, len(mockDestination.Get()))
-	assert.Empty(t, mockDestination.Get()[4].Email)
+	assert.Empty(t, mockDestination.Get()[7].Email)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -868,25 +871,25 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 	body, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	assert.Equal(t, 1462, len(body))
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.gif")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.jpg")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.blah")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
 	require.NoError(t, err)
@@ -901,13 +904,13 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	}()
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
-	imageService := svc.ImageService
+	imageService := image.NewService(&image.FileSystem{
-	imageService.Store = &image.FileSystem{
 		Staging:  "/tmp/remark42/images.staging",
 		Location: "/tmp/remark42/images",
-		MaxSize:  2000,
+	}, image.ServiceParams{
-	}
+		EditDuration: 100 * time.Millisecond,
-	imageService.TTL = 100 * time.Millisecond
+		MaxSize:      2000,
+	})
 
 	svc.privRest.imageService = imageService
 	svc.ImageService = imageService

backend/app/rest/api/rest_public.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bytes"
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
@@ -192,7 +193,6 @@ func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
 			if err != nil {
 				return "info", data, false, err
 			}
-
 			return "info", data, upd, nil
 		}
 	}
@@ -263,11 +263,11 @@ func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request)
 				if e != nil {
 					return nil, e
 				}
+				sinceTime = time.Now()
 				if len(comments) > 0 {
 					sinceTime = comments[0].Timestamp
 					upd = true
 				}
-				sinceTime = time.Now()
 				return encodeJSONWithHTML(comments)
 			})
 			return "last", data, upd, err
@@ -421,22 +421,8 @@ func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 // GET /picture/{user}/{id} - get picture
 func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
-
-	imgContentType := func(img string) string {
-		img = strings.ToLower(img)
-		switch {
-		case strings.HasSuffix(img, ".png"):
-			return "image/png"
-		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
-			return "image/jpeg"
-		case strings.HasSuffix(img, ".gif"):
-			return "image/gif"
-		}
-		return "image/*"
-	}
-
 	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
-	imgRdr, size, err := s.imageService.Load(id)
+	img, err := s.imageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
 		return
@@ -452,16 +438,10 @@ func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer func() {
+	w.Header().Set("Content-Type", s.imageService.ImgContentType(img))
-		if e := imgRdr.Close(); e != nil {
+	w.Header().Set("Content-Length", strconv.Itoa(len(img)))
-			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
-		}
-	}()
-
-	w.Header().Set("Content-Type", imgContentType(id))
-	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
 	w.WriteHeader(http.StatusOK)
-	if _, err = io.Copy(w, imgRdr); err != nil {
+	if _, err = io.Copy(w, bytes.NewReader(img)); err != nil {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -478,7 +458,7 @@ func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
 
 // GET /robots.txt
 func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
-	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/user",
 		"/img", "/avatar", "/picture"}
 	for i := range allowed {
 		allowed[i] = "Allow: /api/v1" + allowed[i]

backend/app/rest/api/rest_public_test.go

@@ -69,7 +69,35 @@ BKT
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
+	assert.Equal(t,
+		`<h1>h1</h1>
+<pre class="chroma">func TestRest_Preview(t *testing.T) {
+srv, ts := prep(t)
+  require.NotNil(t, srv)
+}
+</pre>`,
+		string(b))
+}
+
+func TestRest_PreviewCode(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	text := `BKTgo
+func main(aa string) int {return 0}
+BKT
+`
+	text = strings.Replace(text, "BKT", "```", -1)
+	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+	j = strings.Replace(j, "\n", "\\n", -1)
+
+	resp, err := post(t, ts.URL+"/api/v1/preview", j)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, `<pre class="chroma"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+</pre>`, string(b))
 }
 
 func TestRest_Find(t *testing.T) {
@@ -421,8 +449,8 @@ func TestRest_Counts(t *testing.T) {
 	j := []store.PostInfo{}
 	err = json.Unmarshal(body, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
-		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+		{URL: "https://radio-t.com/blah2", Count: 2}}, j)
 
 	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
 	require.NoError(t, err)
@@ -499,7 +527,7 @@ func TestRest_Config(t *testing.T) {
 	err := json.Unmarshal([]byte(body), &j)
 	assert.NoError(t, err)
 	assert.Equal(t, 300., j["edit_duration"])
-	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
+	assert.EqualValues(t, []interface{}{"a1", "a2"}, j["admins"])
 	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
@@ -570,7 +598,7 @@ func TestRest_InfoStream(t *testing.T) {
 	assert.Equal(t, 200, code)
 	<-done
 
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
 	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
 	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
 	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
@@ -694,16 +722,25 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/user\nAllow: /api/v1/img\n"+
+		"Allow: /api/v1/avatar\nAllow: /api/v1/picture\n", body)
 }
 
 func TestRest_LastCommentsStream(t *testing.T) {
+	t.Skip() // TODO: enable after cache is migrated to https://github.com/dgraph-io/ristretto
 	ts, srv, teardown := startupT(t)
 	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.Refresh = 50 * time.Millisecond
 	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
 	postComment(t, ts.URL)
 
 	defer teardown()
@@ -711,9 +748,10 @@ func TestRest_LastCommentsStream(t *testing.T) {
 	go func() {
 		defer close(done)
 		for i := 1; i < 10; i++ {
-			time.Sleep(100 * time.Millisecond)
 			postComment(t, ts.URL)
+			time.Sleep(100 * time.Millisecond)
 		}
+		t.Log("wrote 10 records")
 	}()
 
 	client := http.Client{}
@@ -758,6 +796,13 @@ func TestRest_LastCommentsStreamCancel(t *testing.T) {
 	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
 	postComment(t, ts.URL)
 
 	defer teardown()
@@ -824,6 +869,13 @@ func TestRest_LastCommentsStreamSince(t *testing.T) {
 	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
 	postComment(t, ts.URL)
 
 	defer teardown()

backend/app/rest/api/rest_test.go

@@ -17,15 +17,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
 	cache "github.com/go-pkgz/lcw"
-	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
@@ -50,12 +49,12 @@ func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	testHtmlName := "test-remark.html"
+	testHTMLName := "test-remark.html"
-	testHTMLFile := os.TempDir() + "/" + testHtmlName
+	testHTMLFile := os.TempDir() + "/" + testHTMLName
 	err := ioutil.WriteFile(testHTMLFile, []byte("some html"), 0700)
 	assert.NoError(t, err)
 
-	body, code := get(t, ts.URL+"/web/"+testHtmlName)
+	body, code := get(t, ts.URL+"/web/"+testHTMLName)
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "some html", body)
 	_ = os.Remove(testHTMLFile)
@@ -234,6 +233,7 @@ func Test_URLKey(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -259,6 +259,7 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -286,6 +287,7 @@ func TestRest_parseError(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
 			res := parseError(tt.err, rest.ErrInternal)
 			assert.Equal(t, tt.res, res)
@@ -309,6 +311,7 @@ func TestRest_cacheControl(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			req := httptest.NewRequest("GET", tt.url, nil)
 			w := httptest.NewRecorder()
@@ -327,8 +330,6 @@ func TestRest_cacheControl(t *testing.T) {
 }
 
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
-	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
-
 	tmp := os.TempDir()
 	var testDb string
 	// pick a file name which is not in use for sure
@@ -345,9 +346,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
 	require.NoError(t, err)
 
-	cacheBackend, err := cache.NewExpirableCache()
+	memCache := cache.NewScache(cache.NewNopCache())
-	require.NoError(t, err)
-	memCache := cache.NewScache(cacheBackend)
 
 	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
@@ -365,21 +364,21 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
+			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
-		Cache:     memCache,
+		Cache:      memCache,
-		WebRoot:   tmp,
+		WebRoot:    tmp,
-		RemarkURL: "https://demo.remark42.com",
+		RemarkURL:  "https://demo.remark42.com",
-		ImageService: &image.Service{
+		AdminEmail: "admin@example.org",
-			Store: &image.FileSystem{
+		ImageService: image.NewService(&image.FileSystem{
-				Location:   tmp + "/pics-remark42",
+			Location:   tmp + "/pics-remark42",
-				Partitions: 100,
+			Partitions: 100,
-				MaxSize:    10000,
+			Staging:    tmp + "/pics-remark42/staging",
-				Staging:    tmp + "/pics-remark42/staging",
+		}, image.ServiceParams{
-			},
+			EditDuration: 100 * time.Millisecond,
-			TTL: time.Millisecond * 100,
+			MaxSize:      10000,
-		},
+		}),
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -388,7 +387,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
-			UrlMapperMaker:    migrator.NewUrlMapper,
+			URLMapperMaker:    migrator.NewURLMapper,
 			Cache:             memCache,
 			KeyStore:          astore,
 		},

backend/app/rest/httperrors.go

@@ -38,7 +38,7 @@ const (
 	ErrAssetNotFound      = 18 // requested file not found
 )
 
-const errorHtml = `<!DOCTYPE html>
+const errorHTML = `<!DOCTYPE html>
 <html>
 <head>
     <meta name="viewport" content="width=device-width"/>
@@ -64,12 +64,12 @@ type errTmplData struct {
 func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
 	// MustExecute behaves like template.Execute, but panics if an error occurs.
 	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
-		if err := tmpl.Execute(wr, data); err != nil {
+		if err = tmpl.Execute(wr, data); err != nil {
 			panic(err)
 		}
 	}
 
-	tmpl := template.Must(template.New("error").Parse(errorHtml))
+	tmpl := template.Must(template.New("error").Parse(errorHTML))
 	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
 	render.Status(r, httpStatusCode)
 	msg := bytes.Buffer{}

backend/app/rest/proxy/image.go

@@ -3,13 +3,10 @@ package proxy
 import (
 	"bytes"
 	"context"
-	"crypto/sha1" // nolint
 	"encoding/base64"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -84,15 +81,6 @@ func (p Image) replace(commentHTML string, imgs []string) string {
 
 // Handler returns http handler respond to proxied request
 func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
-	if !p.HTTP2HTTPS && !p.CacheExternal {
-		// TODO: we might need to find a better way to handle it. If admin enables caching/proxy and disables it later on
-		// all comments that got converted will lose their images. We can't just return a redirect (it will open an ability
-		// to redirect anywhere). We can probably continue proxying these images (but need to make sure this behavior is
-		// documented) or, better, provide a way to migrate back converted comments.
-		http.Error(w, "none of the proxy features are enabled", http.StatusNotImplemented)
-		return
-	}
-
 	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
@@ -100,40 +88,24 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	imgURL := string(src)
-	var imgReader io.ReadCloser
+	var img []byte
-	imgID, err := cachedImgID(imgURL)
+	imgID, err := image.CachedImgID(imgURL)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse image url "+imgURL, rest.ErrAssetNotFound)
 		return
 	}
-	if p.CacheExternal {
+	// try to load from cache for case it was saved when CacheExternal was enabled
-		imgReader, _, err = p.ImageService.Load(imgID)
+	img, _ = p.ImageService.Load(imgID)
-		if err != nil {
+	if img == nil {
-			imgReader = nil
+		img, err = p.downloadImage(context.Background(), imgURL)
-		}
-	}
-	if imgReader == nil {
-		imgReader, err = p.downloadImage(context.Background(), imgURL)
 		if err != nil {
 			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
 			return
 		}
 		if p.CacheExternal {
-			var buf bytes.Buffer
+			p.cacheImage(bytes.NewReader(img), imgID)
-			// We need to duplicate data into a new buffer because `cacheImage` would read provider Reader
-			// and we would need another one to read data for response
-			p.cacheImage(io.TeeReader(imgReader, &buf), imgID)
-			if err := imgReader.Close(); err != nil {
-				log.Printf("[WARN] can't close image reader, %s", err)
-			}
-			imgReader = ioutil.NopCloser(&buf)
 		}
 	}
-	defer func() {
-		if e := imgReader.Close(); e != nil {
-			log.Printf("[WARN] can't close image reader, %s", e)
-		}
-	}()
 
 	// enforce client-side caching
 	etag := `"` + r.URL.Query().Get("src") + `"`
@@ -146,8 +118,8 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	w.Header().Add("Content-Type", "image/*")
+	w.Header().Add("Content-Type", p.ImageService.ImgContentType(img))
-	_, err = io.Copy(w, imgReader)
+	_, err = io.Copy(w, bytes.NewReader(img))
 	if err != nil {
 		log.Printf("[WARN] can't copy image stream, %s", err)
 	}
@@ -155,18 +127,14 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 
 // cache image from provided Reader using given ID
 func (p Image) cacheImage(r io.Reader, imgID string) {
-	id, err := p.ImageService.SaveWithID(imgID, r)
+	err := p.ImageService.SaveWithID(imgID, r)
 	if err != nil {
 		log.Printf("[WARN] unable to save image to the storage: %+v", err)
 	}
-	// In the future we can do something smarter than just committing everything (eg, some kind of LFU/LRU)
-	if err := p.ImageService.Commit(id); err != nil {
-		log.Printf("[WARN] unable to commit image %s", imgID)
-	}
 }
 
-// download an image. Returns a Reader which has to be closed by a caller
+// download an image.
-func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser, error) {
+func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error) {
 	log.Printf("[DEBUG] downloading image %s", imgURL)
 
 	timeout := 60 * time.Second // default
@@ -189,8 +157,7 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser,
 		return e
 	})
 	if err != nil {
-		log.Print(err.Error())
+		return nil, errors.Wrapf(err, "can't download image %s", imgURL)
-		return nil, err
 	}
 
 	if resp.StatusCode != http.StatusOK {
@@ -201,22 +168,5 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser,
 	if err != nil {
 		return nil, errors.Errorf("unable to read image body")
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(imgData)), nil
+	return imgData, nil
-}
-
-func sha1Str(s string) string {
-	return fmt.Sprintf("%x", sha1.Sum([]byte(s))) // nolint
-}
-
-// generates ID for a cached image.
-// ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
-// <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
-// <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
-//                              (as accurate as deduplication based on potentially mutable url can be)
-func cachedImgID(imgURL string) (string, error) {
-	parsedURL, err := url.Parse(imgURL)
-	if err != nil {
-		return "", errors.Wrapf(err, "can parse url %s", imgURL)
-	}
-	return fmt.Sprintf("cached_images/%s-%s", sha1Str(parsedURL.Hostname()), sha1Str(imgURL)), nil
 }

backend/app/rest/proxy/image_test.go

@@ -1,7 +1,6 @@
 package proxy
 
 import (
-	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -16,11 +15,38 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store/image"
 )
 
-func TestPicture_Extract(t *testing.T) {
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestImage_Extract(t *testing.T) {
 	tbl := []struct {
 		inp string
 		res []string
@@ -53,6 +79,7 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{HTTP2HTTPS: true}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			res, err := img.extract(tt.inp, func(src string) bool { return strings.HasPrefix(src, "http://") })
 			assert.NoError(t, err)
@@ -61,7 +88,7 @@ func TestPicture_Extract(t *testing.T) {
 	}
 }
 
-func TestPicture_Replace(t *testing.T) {
+func TestImage_Replace(t *testing.T) {
 	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
 		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
@@ -69,112 +96,164 @@ func TestPicture_Replace(t *testing.T) {
 }
 
 func TestImage_Routes(t *testing.T) {
-	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Times(2).Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 }
 
-func TestImage_Routes_CachingImage(t *testing.T) {
+func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+}
+
+func TestImage_RoutesCachingImage(t *testing.T) {
 	imageStore := image.MockStore{}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
 		RoutePath:     "/api/v1/proxy",
-		ImageService:  &image.Service{Store: &imageStore},
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{MaxSize: 1500}),
 	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	imgURL := httpSrv.URL + "/image/img1.png"
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
 
-	imageStore.On("Load", mock.Anything).Once().Return(nil, int64(0), nil)
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
-	imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Run(func(args mock.Arguments) { _, _ = ioutil.ReadAll(args.Get(1).(io.Reader)) }).Return("", nil)
+	imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
-	imageStore.On("Commit", mock.Anything).Once().Return(nil)
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
 	imageStore.AssertCalled(t, "Load", mock.Anything)
-	imageStore.AssertCalled(t, "SaveWithID", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+sha1Str(imgURL), mock.Anything)
+	imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
-	imageStore.AssertCalled(t, "Commit", mock.Anything)
 }
 
-func TestImage_Routes_Using_Cachded_Image(t *testing.T) {
+func TestImage_RoutesUsingCachedImage(t *testing.T) {
 	imageStore := image.MockStore{}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
 		RoutePath:     "/api/v1/proxy",
-		ImageService:  &image.Service{Store: &imageStore},
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{}),
 	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
-	// In order to validate that cached data is used cache "will return" some other data from what http server would
+	// In order to validate that cached data used cache "will return" some other data from what http server would
-	imageReader := ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("%256s", "X"))))
+	testImage := []byte(fmt.Sprintf("%256s", "X"))
-	imageStore.On("Load", mock.Anything).Once().Return(imageReader, int64(256), nil)
+	imageStore.On("Load", mock.Anything).Once().Return(testImage, nil)
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "256", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.Equal(t, "text/plain; charset=utf-8", resp.Header["Content-Type"][0],
+		"if you save text you receive text/plain in response, that's only fair option you got")
 
 	imageStore.AssertCalled(t, "Load", mock.Anything)
 }
 
 func TestImage_RoutesTimedOut(t *testing.T) {
-	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		Timeout:      50 * time.Millisecond,
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, resp.Body.Close())
 	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
 }
 
-func TestPicture_Convert_ProxyMode(t *testing.T) {
+func TestImage_ConvertProxyMode(t *testing.T) {
 	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
@@ -191,7 +270,7 @@ func TestPicture_Convert_ProxyMode(t *testing.T) {
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
 }
 
-func TestPicture_Convert_CachingMode(t *testing.T) {
+func TestImage_ConvertCachingMode(t *testing.T) {
 	img := Image{CacheExternal: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
@@ -206,19 +285,19 @@ func TestPicture_Convert_CachingMode(t *testing.T) {
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/>`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/>`, r)
 
-	// both Caching and Proxy are enabled
+	// both Caching and Proxy enabled
 	img = Image{CacheExternal: true, HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 }
 
-func imgHTTPServer(t *testing.T) *httptest.Server {
+func imgHTTPTestsServer(t *testing.T) *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/image/img1.png" {
 			t.Log("http img request", r.URL)
-			w.Header().Add("Content-Length", "123")
+			w.Header().Add("Content-Length", "1462")
 			w.Header().Add("Content-Type", "image/png")
-			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write(gopherPNGBytes())
 			assert.NoError(t, err)
 			return
 		}

backend/app/rest/user_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -20,7 +21,7 @@ func TestUser_GetUserInfo(t *testing.T) {
 	assert.Equal(t, store.User{Name: "test", ID: "id", SiteID: "test"}, u)
 }
 
-func TestUSer_MustGetUserInfo(t *testing.T) {
+func TestUser_MustGetUserInfo(t *testing.T) {
 	defer func() {
 		if r := recover(); r != nil {
 			t.Log("recovered from panic")

backend/app/store/admin/admin.go

@@ -79,4 +79,4 @@ func (s *StaticStore) Enabled(site string) (ok bool, err error) {
 }
 
 // OnEvent doesn nothing for StaticStore
-func (s *StaticStore) OnEvent(siteID string, et EventType) error { return nil }
+func (s *StaticStore) OnEvent(_ string, _ EventType) error { return nil }

backend/app/store/admin/remote_test.go

@@ -96,6 +96,6 @@ func testServer(t *testing.T, req, resp string) *httptest.Server {
 		require.NoError(t, err)
 		assert.Equal(t, req, string(body))
 		t.Logf("req: %s", string(body))
-		fmt.Fprintf(w, resp)
+		_, _ = fmt.Fprint(w, resp)
 	}))
 }

backend/app/store/comment.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"fmt"
 	"html/template"
 	"regexp"
 	"strings"
@@ -109,12 +110,20 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 // Sanitize clean dangerous html/js from the comment
 func (c *Comment) Sanitize() {
 	p := bluemonday.UGCPolicy()
-	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
+	p.AllowAttrs("class").Matching(regexp.MustCompile("^chroma$")).OnElements("pre")
+	// this is list of <span> tag classes which could be produced by chroma code renderer
+	// source: https://github.com/alecthomas/chroma/blob/022b6f4fc2c4aa819aac18363c8de3f70619200b/types.go#L221-L316
+	const codeSpanClassRegex = "^(chroma|ln|lnt|hl|lntable|lntd|w|err|x|esc|k|kc" +
+		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
+		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
+		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
+	p.AllowAttrs("class").Matching(regexp.MustCompile(codeSpanClassRegex)).OnElements("span")
 	c.Text = p.Sanitize(c.Text)
 	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = c.escapeHtmlWithSome(c.User.Name)
+	c.User.Name = c.escapeHTMLWithSome(c.User.Name)
-	c.User.Picture = p.Sanitize(c.User.Picture)
+	c.User.Picture = c.SanitizeAsURL(c.User.Picture)
+	c.Locator.URL = c.SanitizeAsURL(c.Locator.URL)
 }
 
 // Snippet from comment's text
@@ -138,7 +147,20 @@ func (c *Comment) Snippet(limit int) string {
 	return string(snippet) + " ..."
 }
 
-func (c *Comment) escapeHtmlWithSome(inp string) string {
+var reHref = regexp.MustCompile(`<a\s+(?:[^>]*?\s+)?href="([^"]*)"`)
+
+// SanitizeAsURL drops dangerous code from a url.
+// It wraps input with href to trigger bluemonday sanitizer and cleans href after sanitizing done
+func (c *Comment) SanitizeAsURL(inp string) string {
+	h := fmt.Sprintf(`<a href="%s">`, inp)
+	clean := bluemonday.UGCPolicy().Sanitize(h)
+	if match := reHref.FindStringSubmatch(clean); len(match) > 1 {
+		return match[1]
+	}
+	return "" // this shouldn't happen as we build the href
+}
+
+func (c *Comment) escapeHTMLWithSome(inp string) string {
 	res := template.HTMLEscapeString(inp)
 	res = strings.Replace(res, "&#34;", "\"", -1)
 	res = strings.Replace(res, "&#39;", "'", -1)

backend/app/store/comment_test.go

@@ -47,6 +47,25 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah & & 123", User: User{Name: "name <> & ' ` \""}},
 			out: Comment{Text: `blah &amp; &amp; 123`, User: User{Name: "name &lt;&gt; & ' ` \""}},
 		},
+
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert('XSS1')"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert(document.domain)//"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "<script>alert()</script>"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: "%3Cscript%3Ealert%28%29%3C/script%3E"}},
+		},
+		{
+			inp: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+			out: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -148,6 +167,7 @@ func TestComment_Snippet(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			c := Comment{Text: tt.inp}
 			out := c.Snippet(tt.limit)
@@ -155,3 +175,40 @@ func TestComment_Snippet(t *testing.T) {
 		})
 	}
 }
+
+func TestComment_SanitizeAsURL(t *testing.T) {
+
+	tbl := []struct {
+		inp, out string
+	}{
+		{
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"javascript:alert(document.domain)//",
+			"",
+		},
+		{
+			"<script>alert()</script>",
+			"%3Cscript%3Ealert%28%29%3C/script%3E",
+		},
+		{
+			"<a href=javascript:alert(document.domain)//>xxx</a>",
+			"",
+		},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		c := Comment{}
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.out, c.SanitizeAsURL(tt.inp))
+		})
+	}
+
+}

backend/app/store/engine/bolt.go

@@ -7,10 +7,10 @@ import (
 	"strings"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -662,7 +662,7 @@ func (b *BoltDB) getUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 		value := bucket.Get([]byte(req.UserID))
 		// return no error in case of absent entry
 		if value != nil {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 			switch req.Detail {
@@ -690,7 +690,7 @@ func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 		value := bucket.Get([]byte(req.UserID))
 		// return no error in case of absent entry
 		if value != nil {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 		}
@@ -711,7 +711,7 @@ func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 	}
 
 	err = bdb.Update(func(tx *bolt.Tx) error {
-		err := b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
+		err = b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
 		return errors.Wrapf(err, "failed to update detail %s for %s in %s", req.Detail, req.UserID, req.Locator.SiteID)
 	})
 
@@ -729,7 +729,7 @@ func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err e
 		var entry UserDetailEntry
 		bucket := tx.Bucket([]byte(userDetailsBucketName))
 		return bucket.ForEach(func(userID, value []byte) error {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 			result = append(result, entry)
@@ -860,6 +860,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
 	// get list of commentID for all user's comment
 	comments := []commentInfo{}
 	for _, postInfo := range posts {
+		postInfo := postInfo
 		err = bdb.View(func(tx *bolt.Tx) error {
 			postsBkt := tx.Bucket([]byte(postsBucketName))
 			postBkt := postsBkt.Bucket([]byte(postInfo.URL))

backend/app/store/engine/bolt_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 )

backend/app/store/engine/engine.go

@@ -16,18 +16,21 @@ import (
 
 // Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Create(comment store.Comment) (commentID string, err error)  // create new comment, avoid dups by id
+	Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
-	Update(comment store.Comment) error                          // update comment, mutable parts only
+	Update(comment store.Comment) error                         // update comment, mutable parts only
-	Get(req GetRequest) (store.Comment, error)                   // get comment by id
+	Get(req GetRequest) (store.Comment, error)                  // get comment by id
-	Find(req FindRequest) ([]store.Comment, error)               // find comments for locator or site
+	Find(req FindRequest) ([]store.Comment, error)              // find comments for locator or site
-	Info(req InfoRequest) ([]store.PostInfo, error)              // get post(s) meta info
+	Info(req InfoRequest) ([]store.PostInfo, error)             // get post(s) meta info
-	Count(req FindRequest) (int, error)                          // get count for post or user
+	Count(req FindRequest) (int, error)                         // get count for post or user
-	Delete(req DeleteRequest) error                              // Delete post(s), user, comment, user details, or everything
+	Delete(req DeleteRequest) error                             // Delete post(s), user, comment, user details, or everything
-	Flag(req FlagRequest) (bool, error)                          // set and get flags
+	Flag(req FlagRequest) (bool, error)                         // set and get flags
-	ListFlags(req FlagRequest) ([]interface{}, error)            // get list of flagged keys, like blocked & verified user
+	ListFlags(req FlagRequest) ([]interface{}, error)           // get list of flagged keys, like blocked & verified user
-	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) // sets or gets single detail value, or gets all details for requested site.
+
-	// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+	// UserDetail sets or gets single detail value, or gets all details for requested site
-	// and all site's details listing under the same function (and not to extend interface by two separate functions).
+	// Returns list even for single entry request is a compromise in order to have both single detail getting and setting
+	// and all site's details listing under the same function (and not to extend interface by two separate functions)
+	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error)
+
 	Close() error // close storage engine
 }
 
@@ -83,10 +86,13 @@ const (
 	Verified = Flag("verified")
 	Blocked  = Flag("blocked")
 )
+
+// All possible user details
 const (
-	// All possible user details
+	// UserEmail is a user email
-	UserEmail      = UserDetail("email")
+	UserEmail = UserDetail("email")
-	AllUserDetails = UserDetail("all") // used for listing and deletion requests
+	// AllUserDetails used for listing and deletion requests
+	AllUserDetails = UserDetail("all")
 )
 
 // FlagRequest is the input for both get/set for flags, like blocked, verified and so on

backend/app/store/engine/engine_mock.go

@@ -2,10 +2,8 @@
 
 package engine
 
-import (
+import mock "github.com/stretchr/testify/mock"
-	mock "github.com/stretchr/testify/mock"
+import store "github.com/umputun/remark/backend/app/store"
-	store "github.com/umputun/remark/backend/app/store"
-)
 
 // MockInterface is an autogenerated mock type for the Interface type
 type MockInterface struct {

backend/app/store/formatter.go

@@ -4,8 +4,10 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/Depado/bfchroma"
 	"github.com/PuerkitoBio/goquery"
-	bf "gopkg.in/russross/blackfriday.v2"
+	"github.com/alecthomas/chroma/formatters/html"
+	bf "github.com/russross/blackfriday/v2"
 )
 
 // CommentFormatter implements all generic formatting ops on comment
@@ -48,7 +50,9 @@ func (f *CommentFormatter) FormatText(txt string) (res string) {
 		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
 	})
 
-	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
+	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
+
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(extRend)))
 	res = f.unEscape(res)
 
 	for _, conv := range f.converters {

backend/app/store/formatter_test.go

@@ -25,9 +25,15 @@ func TestFormatter_FormatText(t *testing.T) {
 				"1/some-long-link/12345/6789012...</a></p>\n!converted", "links",
 		},
 		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
+		{"smth\n```go\nfunc main(aa string) int {return 0}\n```", `<p>smth</p>
+<pre class="chroma"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+</pre>!converted`, "code with language"},
+		{"```\ntest_code\n```", `<pre class="chroma">test_code
+</pre>!converted`, "code without language"},
 	}
 	f := NewCommentFormatter(mockConverter{})
 	for _, tt := range tbl {
+		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			assert.Equal(t, tt.out, f.FormatText(tt.in))
 		})

backend/app/store/image/bolt_store.go

@@ -4,14 +4,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
-	"io"
-	"io/ioutil"
-	"path"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 const imagesStagedBktName = "imagesStaged"
@@ -22,15 +19,12 @@ const insertTimeBktName = "insertTimestamps"
 // It uses 3 buckets to manage images data.
 // Two buckets contains image data (staged and committed images). Third bucket holds insertion timestamps.
 type Bolt struct {
-	fileName  string
+	fileName string
-	db        *bolt.DB
+	db       *bolt.DB
-	MaxSize   int
-	MaxHeight int
-	MaxWidth  int
 }
 
-// Create Bolt Store.
+// NewBoltStorage create bolt image store
-func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, options bolt.Options) (*Bolt, error) {
+func NewBoltStorage(fileName string, options bolt.Options) (*Bolt, error) {
 	db, err := bolt.Open(fileName, 0600, &options)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
@@ -52,48 +46,32 @@ func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, o
 		return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
 	}
 	return &Bolt{
-		db:        db,
+		db:       db,
-		fileName:  fileName,
+		fileName: fileName,
-		MaxSize:   maxSize,
-		MaxHeight: maxHeight,
-		MaxWidth:  maxWidth,
 	}, nil
 }
 
-// SaveWithID saves data from reader with given id
+// Save saves image for given id to staging bucket in DB
-func (b *Bolt) SaveWithID(id string, r io.Reader) (string, error) {
+func (b *Bolt) Save(id string, img []byte) error {
-	data, err := readAndValidateImage(r, b.MaxSize)
+	err := b.db.Update(func(tx *bolt.Tx) error {
-	if err != nil {
+		if err := tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), img); err != nil {
-		return "", errors.Wrapf(err, "can't load image with ID %s", id)
-	}
-
-	data = resize(data, b.MaxWidth, b.MaxHeight)
-
-	err = b.db.Update(func(tx *bolt.Tx) error {
-		if err = tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), data); err != nil {
 			return errors.Wrapf(err, "can't put to bucket with %s", id)
 		}
 		tsBuf := &bytes.Buffer{}
-		if err = binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil {
+		if err := binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil {
 			return errors.Wrapf(err, "can't serialize timestamp for %s", id)
 		}
-		if err = tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil {
+		if err := tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil {
 			return errors.Wrapf(err, "can't put to bucket with %s", id)
 		}
-		return err
+		return nil
 	})
 
-	return id, err
+	return err
-}
-
-// Save data from reader to staging bucket in DB
-func (b *Bolt) Save(fileName string, userID string, r io.Reader) (id string, err error) {
-	id = path.Join(userID, guid())
-	return b.SaveWithID(id, r)
 }
 
 // Commit file stored in staging bucket by copying it to permanent bucket
-// Data from staging bucket not removed immediately, but would be removed on cleanup
+// Data from staging bucket not removed immediately, but would be removed on Cleanup
 func (b *Bolt) Commit(id string) error {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
@@ -107,31 +85,31 @@ func (b *Bolt) Commit(id string) error {
 }
 
 // Load image from DB
-// returns ReadCloser and caller should call close after processing completed.
+func (b *Bolt) Load(id string) ([]byte, error) {
-func (b *Bolt) Load(id string) (io.ReadCloser, int64, error) {
+	var data []byte
-	buf := &bytes.Buffer{}
-	var size int = 0
 	err := b.db.View(func(tx *bolt.Tx) error {
-		data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
+		data = tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
 		if data == nil {
 			data = tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
 		}
 		if data == nil {
 			return errors.Errorf("can't load image %s", id)
 		}
-		var err error
+		return nil
-		size, err = buf.Write(data)
-		return errors.Wrapf(err, "failed to write for %s", id)
 	})
-	return ioutil.NopCloser(buf), int64(size), err
+	if err != nil {
+		// separate error handler to return nil and not empty []byte
+		return nil, err
+	}
+	return data, nil
 }
 
 // Cleanup runs scan of staging and removes old data based on ttl
-func (b *Bolt) Cleanup(ctx context.Context, ttl time.Duration) error {
+func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		c := tx.Bucket([]byte(insertTimeBktName)).Cursor()
 
-		idsToRemove := [][]byte{}
+		var idsToRemove [][]byte
 
 		for id, tsData := c.First(); id != nil; id, tsData = c.Next() {
 			var ts int64
@@ -163,7 +141,25 @@ func (b *Bolt) Cleanup(ctx context.Context, ttl time.Duration) error {
 	return err
 }
 
-// SizeLimit returns max size of allowed image
+// Info returns meta information about storage
-func (b *Bolt) SizeLimit() int {
+func (b *Bolt) Info() (StoreInfo, error) {
-	return b.MaxSize
+	var ts time.Time
+	err := b.db.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(insertTimeBktName)).Cursor()
+
+		for id, tsData := c.First(); id != nil; id, tsData = c.Next() {
+			var createdRaw int64
+			err := binary.Read(bytes.NewReader(tsData), binary.LittleEndian, &createdRaw)
+			if err != nil {
+				return errors.Wrapf(err, "failed to deserialize timestamp for %s", id)
+			}
+
+			created := time.Unix(0, createdRaw)
+			if ts.IsZero() || created.Before(ts) {
+				ts = created
+			}
+		}
+		return nil
+	})
+	return StoreInfo{FirstStagingImageTS: ts}, errors.Wrapf(err, "problem retrieving first timestamp from staging images")
 }

backend/app/store/image/bolt_store_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -18,10 +18,10 @@ func TestBoltStore_SaveCommit(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	id := "test_img"
+
+	err := svc.Save(id, gopherPNGBytes())
 	assert.NoError(t, err)
-	assert.Contains(t, id, "user1")
-	t.Log(id)
 
 	err = svc.db.View(func(tx *bolt.Tx) error {
 		data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
@@ -48,21 +48,16 @@ func TestBoltStore_LoadAfterSave(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
 	assert.NoError(t, err)
-	assert.Contains(t, id, "user1")
-	t.Log(id)
-
-	r, sz, err := svc.Load(id)
-	assert.NoError(t, err)
-	defer func() { assert.NoError(t, r.Close()) }()
-	data, err := ioutil.ReadAll(r)
 
+	data, err := svc.Load(id)
 	assert.NoError(t, err)
 	assert.Equal(t, 1462, len(data))
-	assert.Equal(t, int64(1462), sz)
+	assert.Equal(t, gopherPNGBytes(), data)
 
-	_, _, err = svc.Load("abcd")
+	_, err = svc.Load("abcd")
 	assert.Error(t, err)
 }
 
@@ -70,25 +65,25 @@ func TestBoltStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()
 
-	save := func(file string, user string) (id string) {
+	save := func(file string) (id string) {
-		id, err := svc.Save(file, user, gopherPNG())
+		err := svc.Save(file, gopherPNGBytes())
 		require.NoError(t, err)
 
-		checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error {
+		checkBoltImgData(t, svc.db, imagesStagedBktName, file, func(data []byte) error {
 			require.NotNil(t, data)
 			assert.Equal(t, 1462, len(data))
 			return nil
 		})
-		return id
+		return file
 	}
 
 	// save 3 images to staging
-	img1 := save("blah_ff1.png", "user1")
+	img1 := save("blah_ff1.png")
 	img1ts := time.Now()
 	time.Sleep(100 * time.Millisecond)
-	img2 := save("blah_ff2.png", "user1")
+	img2 := save("blah_ff2.png")
 	time.Sleep(100 * time.Millisecond)
-	img3 := save("blah_ff3.png", "user2")
+	img3 := save("blah_ff3.png")
 
 	err := svc.Cleanup(context.Background(), time.Since(img1ts)) // clean first images
 	assert.NoError(t, err)
@@ -110,6 +105,25 @@ func TestBoltStore_Cleanup(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestBolt_Info(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}
+
 func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) {
 	checkBoltImgData(t, db, bucket, id, func(data []byte) error {
 		assert.Nil(t, data, id)
@@ -138,7 +152,7 @@ func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) {
 	loc, err := ioutil.TempDir("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
 
-	svc, err = NewBoltStorage(path.Join(loc, "picture.db"), 1500, 0, 0, bolt.Options{})
+	svc, err = NewBoltStorage(path.Join(loc, "picture.db"), bolt.Options{})
 	assert.NoError(t, err, "new bolt storage")
 
 	teardown = func() {

backend/app/store/image/fs_store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"hash/crc64"
-	"io"
 	"io/ioutil"
 	"math"
 	"os"
@@ -23,10 +22,7 @@ import (
 type FileSystem struct {
 	Location   string
 	Staging    string
-	MaxSize    int
 	Partitions int
-	MaxHeight  int
-	MaxWidth   int
 
 	crc struct {
 		*crc64.Table
@@ -36,42 +32,26 @@ type FileSystem struct {
 	}
 }
 
-// SaveWithID saves data from reader with given id
+// Save saves image with given id to local FS, staging directory.
-func (f *FileSystem) SaveWithID(id string, r io.Reader) (string, error) {
+// Files partitioned across multiple subdirectories, and the final path includes part, i.e. /location/user1/03/123-4567
-	data, err := readAndValidateImage(r, f.MaxSize)
+func (f *FileSystem) Save(id string, img []byte) error {
-	if err != nil {
-		return "", errors.Wrapf(err, "can't load image with ID %s", id)
-	}
-
-	data = resize(data, f.MaxWidth, f.MaxHeight)
 	dst := f.location(f.Staging, id)
 
-	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+	if err := os.MkdirAll(path.Dir(dst), 0700); err != nil {
-		return "", errors.Wrap(err, "can't make image directory")
+		return errors.Wrap(err, "can't make image directory")
 	}
 
-	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
+	if err := ioutil.WriteFile(dst, img, 0600); err != nil {
-		return "", errors.Wrapf(err, "can't write file")
+		return errors.Wrapf(err, "can't write image file with id %s", id)
 	}
 
-	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(data))
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(img))
-	return id, nil
+	return nil
-}
-
-// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid
-// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567
-func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
-	id = path.Join(userID, guid()) // make id as user/uuid
-	finalID, err := f.SaveWithID(id, r)
-	if err != nil {
-		err = errors.Wrapf(err, "can't save file %s", fileName)
-	}
-	return finalID, err
 }
 
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
-	log.Printf("[DEBUG] commit image %s", id)
+	log.Printf("[DEBUG] Commit image %s", id)
 	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
 
 	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
@@ -83,39 +63,39 @@ func (f *FileSystem) Commit(id string) error {
 }
 
 // Load image from FS. Uses id to get partition subdirectory.
-// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) ([]byte, error) {
-func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 
 	// get image file by id. first try permanent location and if not found - staging
-	img := func(id string) (file string, st os.FileInfo, err error) {
+	img := func(id string) (file string, err error) {
 		file = f.location(f.Location, id)
-		st, err = os.Stat(file)
+		_, err = os.Stat(file)
 		if err != nil {
 			file = f.location(f.Staging, id)
-			st, err = os.Stat(file)
+			_, err = os.Stat(file)
 		}
-		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+		return file, errors.Wrapf(err, "can't get image stats for %s", id)
 	}
 
-	imgFile, st, err := img(id)
+	imgFile, err := img(id)
 	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+		return nil, errors.Wrapf(err, "can't get image file for %s", id)
 	}
 
-	fh, err := os.Open(imgFile)
+	fh, err := os.Open(imgFile) //nolint:gosec
 	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+		return nil, errors.Wrapf(err, "can't load image %s", id)
 	}
-	return fh, st.Size(), nil
+	return ioutil.ReadAll(fh)
 }
 
 // Cleanup runs scan of staging and removes old files based on ttl
-func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error {
 
 	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
 		return nil
 	}
 
+	// we can ignore context as on local FS remove is relatively fast operation
 	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -124,7 +104,7 @@ func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
 			return nil
 		}
 		age := time.Since(info.ModTime())
-		if age > ttl {
+		if age > (ttl + 100*time.Millisecond) { // delay cleanup triggering to allow commit
 			log.Printf("[INFO] remove staging image %s, age %v", fpath, age)
 			rmErr := os.Remove(fpath)
 			_ = os.Remove(path.Dir(fpath)) // try to remove directory
@@ -135,9 +115,31 @@ func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
 	return errors.Wrap(err, "failed to cleanup images")
 }
 
-// SizeLimit returns max size of allowed image
+// Info returns meta information about storage
-func (f *FileSystem) SizeLimit() int {
+func (f *FileSystem) Info() (StoreInfo, error) {
-	return f.MaxSize
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return StoreInfo{}, nil
+	}
+
+	var ts time.Time
+	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		created := info.ModTime()
+		if ts.IsZero() || created.Before(ts) {
+			ts = created
+		}
+		return nil
+	})
+	if err != nil {
+		return StoreInfo{}, errors.Wrapf(err, "problem retrieving first timestamp from staging images on fs")
+	}
+	return StoreInfo{FirstStagingImageTS: ts}, nil
 }
 
 // location gets full path for id by adding partition to the final path in order to keep files in different subdirectories

backend/app/store/image/fs_store_test.go

@@ -39,95 +39,51 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
 	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
 func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
 
 func TestFsStore_Save(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
 	assert.NoError(t, err)
-	assert.Contains(t, id, "user1/")
-	t.Log(id)
 
 	img := svc.location(svc.Staging, id)
-	t.Log(img)
 	data, err := ioutil.ReadFile(img)
 	assert.NoError(t, err)
 	assert.Equal(t, 1462, len(data))
 }
 
-func TestFsStore_SaveWithResize(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-	svc.MaxWidth, svc.MaxHeight = 32, 32
-
-	id, err := svc.Save("file1.png", "user1", gopherPNG())
-	assert.NoError(t, err)
-	assert.Contains(t, id, "user1/")
-	t.Log(id)
-
-	img := svc.location(svc.Staging, id)
-	t.Log(img)
-	data, err := ioutil.ReadFile(img)
-	assert.NoError(t, err)
-	assert.Equal(t, 1142, len(data))
-}
-
-func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-	svc.MaxWidth, svc.MaxHeight = 400, 300
-	svc.MaxSize = 32000
-
-	fh, err := os.Open("testdata/circles.jpg")
-	defer func() { assert.NoError(t, fh.Close()) }()
-	assert.NoError(t, err)
-	id, err := svc.Save("circles.jpg", "user1", fh)
-	assert.NoError(t, err)
-	assert.Contains(t, id, "user1/")
-	t.Log(id)
-
-	img := svc.location(svc.Staging, id)
-	t.Log(img)
-	data, err := ioutil.ReadFile(img)
-	assert.NoError(t, err)
-	assert.Equal(t, 10786, len(data))
-}
-
 func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
-	svc.MaxWidth, svc.MaxHeight = 1400, 1300
-	svc.MaxSize = 32000
 
 	fh, err := os.Open("testdata/circles.jpg")
 	defer func() { assert.NoError(t, fh.Close()) }()
 	assert.NoError(t, err)
-	id, err := svc.Save("circles.jpg", "user1", fh)
+	img, err := ioutil.ReadAll(fh)
+	assert.NoError(t, err)
+	id := "test_img"
+	err = svc.Save(id, img)
 	assert.NoError(t, err)
-	assert.Contains(t, id, "user1/")
-	t.Log(id)
 
-	img := svc.location(svc.Staging, id)
+	imgPath := svc.location(svc.Staging, id)
-	t.Log(img)
+	t.Log(imgPath)
-	data, err := ioutil.ReadFile(img)
+	data, err := ioutil.ReadFile(imgPath)
 	assert.NoError(t, err)
 	assert.Equal(t, 23983, len(data))
 }
 
-func TestFsStore_WrongFormat(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
-	assert.Error(t, err)
-}
-
 func TestFsStore_SaveAndCommit(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
 	require.NoError(t, err)
 	err = svc.Commit(id)
 	require.NoError(t, err)
@@ -143,32 +99,20 @@ func TestFsStore_SaveAndCommit(t *testing.T) {
 	assert.Equal(t, 1462, len(data))
 }
 
-func TestFsStore_SaveTooLarge(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-	svc.MaxSize = 2000
-	_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
-	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "is too large")
-}
-
 func TestFsStore_LoadAfterSave(t *testing.T) {
 
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
 	assert.NoError(t, err)
-	t.Log(id)
 
-	r, sz, err := svc.Load(id)
+	data, err := svc.Load(id)
-	assert.NoError(t, err)
-	defer func() { assert.NoError(t, r.Close()) }()
-	data, err := ioutil.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, 1462, len(data))
-	assert.Equal(t, int64(1462), sz)
+	assert.Equal(t, gopherPNGBytes(), data)
-	_, _, err = svc.Load("abcd")
+	_, err = svc.Load("abcd")
 	assert.Error(t, err)
 }
 
@@ -177,20 +121,16 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
 	assert.NoError(t, err)
-	t.Log(id)
 	err = svc.Commit(id)
 	require.NoError(t, err)
 
-	r, sz, err := svc.Load(id)
+	data, err := svc.Load(id)
-	assert.NoError(t, err)
-	defer func() { assert.NoError(t, r.Close()) }()
-	data, err := ioutil.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, 1462, len(data))
-	assert.Equal(t, int64(1462), sz)
+	_, err = svc.Load("abcd")
-	_, _, err = svc.Load("abcd")
 	assert.Error(t, err)
 }
 
@@ -211,6 +151,7 @@ func TestFsStore_location(t *testing.T) {
 		{0, "user/12345", "/tmp/user/12345"},
 	}
 	for n, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
 			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
 			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
@@ -242,8 +183,9 @@ func TestFsStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	save := func(file string, user string) (path string) {
+	save := func(file string, user string) (filePath string) {
-		id, err := svc.Save(file, user, gopherPNG())
+		id := path.Join(user, file)
+		err := svc.Save(id, gopherPNGBytes())
 		require.NoError(t, err)
 		img := svc.location(svc.Staging, id)
 		data, err := ioutil.ReadFile(img)
@@ -259,14 +201,14 @@ func TestFsStore_Cleanup(t *testing.T) {
 	time.Sleep(100 * time.Millisecond)
 	img3 := save("blah_ff3.png", "user2")
 
-	time.Sleep(100 * time.Millisecond) // make first image expired
+	time.Sleep(200 * time.Millisecond) // make first image expired
 	err := svc.Cleanup(context.Background(), time.Millisecond*300)
 	assert.NoError(t, err)
 
 	_, err = os.Stat(img1)
 	assert.Error(t, err, "no file on staging anymore")
 	// sometimes two images for user1 are put into same directory, which means that
-	// after first image cleanup it's not empty and won't be deleted
+	// after first image Cleanup it's not empty and won't be deleted
 	_, err = os.Stat(path.Dir(img1))
 	if path.Dir(img1) != path.Dir(img2) {
 		assert.Error(t, err, "no dir %s on staging anymore", path.Dir(img1))
@@ -289,6 +231,25 @@ func TestFsStore_Cleanup(t *testing.T) {
 	assert.Error(t, err, "no file on staging anymore")
 }
 
+func TestFsStore_Info(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	// get ts on empty storage, should be zero
+	ts, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, ts.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get ts after saving, should be non-zero
+	ts, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, ts.FirstStagingImageTS.IsZero())
+}
+
 func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
 	loc, err := ioutil.TempDir("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
@@ -300,7 +261,6 @@ func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
 		Location:   loc,
 		Staging:    staging,
 		Partitions: 100,
-		MaxSize:    1500,
 	}
 
 	teardown = func() {

backend/app/store/image/image.go

@@ -1,6 +1,6 @@
 // Package image handles storing, resizing and retrieval of images
-// Provides Store with Save and Load and one implementation on top of local file system.
+// Provides Store with Save and Load implementations on top of local file system and bolt db.
-// Service object encloses Store and add common methods, this is the one consumer should use
+// Service object encloses Store and add common methods, this is the one consumer should use.
 package image
 
 //go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
@@ -8,13 +8,19 @@ package image
 import (
 	"bytes"
 	"context"
+	"crypto/sha1" //nolint:gosec // not used for cryptography
+	"encoding/base64"
+	"fmt"
 	"image"
+	// support gif and jpeg images decoding
 	_ "image/gif"
 	_ "image/jpeg"
 	"image/png"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
+	"path"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -27,27 +33,55 @@ import (
 	"golang.org/x/image/draw"
 )
 
-// Store defines interface for saving and loading pictures.
+// Service wraps Store with common functions needed for any store implementation
-// Declares two-stage save with commit
+// It also provides async Submit with func param retrieving all submitting ids.
-type Store interface {
+// Submitted ids committed (i.e. moved from staging to final) on commitTTL expiration.
-	SaveWithID(id string, r io.Reader) (string, error)
+type Service struct {
-	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	ServiceParams
-	Commit(id string) error                                                  // move image from staging to permanent
+
-	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	store       Store
-	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+	wg          sync.WaitGroup
-	SizeLimit() int                                                          // max image size
+	submitCh    chan submitReq
+	once        sync.Once
+	term        int32 // term value used atomically to detect emergency termination
+	submitCount int32 // atomic increment for counting submitted images
 }
 
-// Service extends Store with common functions needed for any store implementation
+// ServiceParams contains externally adjustable parameters of Service
-type Service struct {
+type ServiceParams struct {
-	Store
+	EditDuration time.Duration // edit period for comments
-	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI     string        // image api matching path
-	ImageAPI string        // image api matching path
+	ProxyAPI     string        // proxy api matching path
+	MaxSize      int
+	MaxHeight    int
+	MaxWidth     int
 
-	wg       sync.WaitGroup
+	// duration of time after which images are checked and committed if still
-	submitCh chan submitReq
+	// present in the submitted comment after it's EditDuration is expired
-	once     sync.Once
+	commitTTL time.Duration
-	term     int32
+	// duration of time after which images are deleted from staging
+	cleanupTTL time.Duration
+}
+
+// StoreInfo contains image store meta information
+type StoreInfo struct {
+	FirstStagingImageTS time.Time
+}
+
+// To regenerate mock run from this directory:
+// sh -c "mockery -inpkg -name Store -print > /tmp/image-mock.tmp && mv /tmp/image-mock.tmp image_mock.go"
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with Commit. Save stores to staging area and Commit moves to the final location.
+// Two-stage commit scheme is used for not storing images which are uploaded but later never used in the comments,
+// e.g. when somebody uploaded a picture but did not sent the comment.
+type Store interface {
+	Info() (StoreInfo, error)         // get meta information about storage
+	Save(id string, img []byte) error // store image with passed id to staging
+	Load(id string) ([]byte, error)   // load image by ID
+
+	Commit(id string) error                               // move image from staging to permanent
+	Cleanup(ctx context.Context, ttl time.Duration) error // run removal loop for old images on staging
 }
 
 const submitQueueSize = 5000
@@ -57,6 +91,15 @@ type submitReq struct {
 	TS    time.Time
 }
 
+// NewService returns new Service instance
+func NewService(s Store, p ServiceParams) *Service {
+	p.commitTTL = p.EditDuration * 15 / 10  // Commit call on every 1.5 * EditDuration
+	p.cleanupTTL = p.EditDuration * 25 / 10 // Cleanup call on every 2.5 * EditDuration
+	// In case Cleanup and Submit start at the same time (case of stale staging images check
+	// on the program start) these TTL values guarantee that Commit will happen before Cleanup.
+	return &Service{ServiceParams: p, store: s}
+}
+
 // Submit multiple ids via function for delayed commit
 func (s *Service) Submit(idsFn func() []string) {
 	if idsFn == nil || s == nil {
@@ -70,20 +113,22 @@ func (s *Service) Submit(idsFn func() []string) {
 		go func() {
 			defer s.wg.Done()
 			for req := range s.submitCh {
-				// wait for TTL expiration with emergency pass on term
+				// wait for commitTTL expiration with emergency pass on term
-				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
 				for _, id := range req.idsFn() {
-					if err := s.Commit(id); err != nil {
+					if err := s.store.Commit(id); err != nil {
 						log.Printf("[WARN] failed to commit image %s", id)
 					}
 				}
+				atomic.AddInt32(&s.submitCount, -1)
 			}
 			log.Printf("[INFO] image submitter terminated")
 		}()
 	})
 
+	atomic.AddInt32(&s.submitCount, 1)
 	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
 }
 
@@ -94,51 +139,128 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error)
 	if err != nil {
 		return nil, errors.Wrap(err, "can't create document")
 	}
-	result := []string{}
 	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
 		if im, ok := sl.Attr("src"); ok {
 			if strings.Contains(im, s.ImageAPI) {
 				elems := strings.Split(im, "/")
 				if len(elems) >= 2 {
 					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
-					result = append(result, id)
+					ids = append(ids, id)
 				}
 			}
+			if strings.Contains(im, s.ProxyAPI) {
+				proxiedURL, err := url.Parse(im)
+				if err != nil {
+					return
+				}
+				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
+				if err != nil {
+					return
+				}
+				imgID, err := CachedImgID(string(imgURL))
+				if err != nil {
+					return
+				}
+				ids = append(ids, imgID)
+			}
 		}
 	})
 
-	return result, nil
+	return ids, nil
 }
 
-// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+// Cleanup runs periodic cleanup with cleanupTTL. Blocking loop, should be called inside of goroutine by consumer
 func (s *Service) Cleanup(ctx context.Context) {
-	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.cleanupTTL)
 
 	for {
 		select {
 		case <-ctx.Done():
 			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
 			return
-		case <-time.After(s.TTL / 2):
+		case <-time.After(s.cleanupTTL):
-			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+			if err := s.store.Cleanup(ctx, s.cleanupTTL); err != nil {
 				log.Printf("[WARN] failed to cleanup, %v", err)
 			}
 		}
 	}
 }
 
+// Info returns meta information about storage
+func (s *Service) Info() (StoreInfo, error) {
+	return s.store.Info()
+}
+
 // Close flushes all in-progress submits and enforces waiting commits
-func (s *Service) Close() {
+func (s *Service) Close(ctx context.Context) {
 	log.Printf("[INFO] close image service ")
-	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+
+	waitForTerm := func(ctx context.Context) {
+		ticker := time.NewTicker(10 * time.Millisecond)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if atomic.LoadInt32(&s.submitCount) == 0 {
+					return
+				}
+			}
+		}
+	}
+
+	atomic.StoreInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	waitForTerm(ctx)
+
 	if s.submitCh != nil {
 		close(s.submitCh)
 	}
 	s.wg.Wait()
 }
 
-// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
+// Load wraps storage Load function.
-// biggest side (width or height) preserving aspect ratio.
+func (s *Service) Load(id string) ([]byte, error) {
+	return s.store.Load(id)
+}
+
+// Save wraps storage Save function, validating and resizing the image before calling it.
+func (s *Service) Save(userID string, r io.Reader) (id string, err error) {
+	id = path.Join(userID, guid())
+	return id, s.SaveWithID(id, r)
+}
+
+// SaveWithID wraps storage Save function, validating and resizing the image before calling it.
+func (s *Service) SaveWithID(id string, r io.Reader) error {
+	img, err := s.prepareImage(r)
+	if err != nil {
+		return err
+	}
+	return s.store.Save(id, img)
+}
+
+// ImgContentType returns content type for provided image
+func (s *Service) ImgContentType(img []byte) string {
+	contentType := http.DetectContentType(img)
+	if contentType == "application/octet-stream" {
+		// replace generic fallback with one which make sense in our scenario
+		return "image/*"
+	}
+	return contentType
+}
+
+// prepareImage calls readAndValidateImage and resize on provided image.
+func (s *Service) prepareImage(r io.Reader) ([]byte, error) {
+	data, err := readAndValidateImage(r, s.MaxSize)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't load image")
+	}
+
+	data = resize(data, s.MaxWidth, s.MaxHeight)
+	return data, nil
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of
+// the biggest side (width or height) preserving aspect ratio.
 // Returns original data if resizing is not needed or failed.
 // If resized the result will be for png format
 func resize(data []byte, limitW, limitH int) []byte {
@@ -161,7 +283,7 @@ func resize(data []byte, limitW, limitH int) []byte {
 
 	newW, newH := getProportionalSizes(w, h, limitW, limitH)
 	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
-	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+	draw.CatmullRom.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
 
 	var out bytes.Buffer
 	if err = png.Encode(&out, m); err != nil {
@@ -191,13 +313,14 @@ func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
 	return limitW, int(propH)
 }
 
-// check if file f is a valid image format, i.e. gif, png, jpeg or webp
+// check if file f is a valid image format, i.e. gif, png, jpeg or webp and reads up to maxSize.
-func isValidImage(b []byte) bool {
-	ct := http.DetectContentType(b)
-	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
-}
-
 func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) {
+
+	isValidImage := func(b []byte) bool {
+		ct := http.DetectContentType(b)
+		return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
+	}
+
 	lr := io.LimitReader(r, int64(maxSize)+1)
 	data, err := ioutil.ReadAll(lr)
 	if err != nil {
@@ -210,7 +333,7 @@ func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) {
 
 	// read header first, needs it to check if data is valid png/gif/jpeg
 	if !isValidImage(data[:512]) {
-		return nil, errors.Errorf("file format is not allowed")
+		return nil, errors.Errorf("file format not allowed")
 	}
 
 	return data, nil
@@ -220,3 +343,21 @@ func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) {
 func guid() string {
 	return xid.New().String()
 }
+
+// Sha1Str converts provided string to sha1
+func Sha1Str(s string) string {
+	return fmt.Sprintf("%x", sha1.Sum([]byte(s))) //nolint:gosec // not used for cryptography
+}
+
+// CachedImgID generates ID for a cached image.
+// ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
+// <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
+// <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
+//                              (as accurate as deduplication based on potentially mutable url can be)
+func CachedImgID(imgURL string) (string, error) {
+	parsedURL, err := url.Parse(imgURL)
+	if err != nil {
+		return "", errors.Wrapf(err, "can parse url %s", imgURL)
+	}
+	return fmt.Sprintf("cached_images/%s-%s", Sha1Str(parsedURL.Hostname()), Sha1Str(imgURL)), nil
+}

backend/app/store/image/image_mock.go

@@ -2,14 +2,9 @@
 
 package image
 
-import (
+import context "context"
-	context "context"
+import mock "github.com/stretchr/testify/mock"
-	io "io"
+import time "time"
-
-	mock "github.com/stretchr/testify/mock"
-
-	time "time"
-)
 
 // MockStore is an autogenerated mock type for the Store type
 type MockStore struct {
@@ -44,50 +39,43 @@ func (_m *MockStore) Commit(id string) error {
 	return r0
 }
 
+// Info provides a mock function with given fields:
+func (_m *MockStore) Info() (StoreInfo, error) {
+	ret := _m.Called()
+
+	var r0 StoreInfo
+	if rf, ok := ret.Get(0).(func() StoreInfo); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(StoreInfo)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Load provides a mock function with given fields: id
-func (_m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+func (_m *MockStore) Load(id string) ([]byte, error) {
 	ret := _m.Called(id)
 
-	var r0 io.ReadCloser
+	var r0 []byte
-	if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
+	if rf, ok := ret.Get(0).(func(string) []byte); ok {
 		r0 = rf(id)
 	} else {
 		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(io.ReadCloser)
+			r0 = ret.Get(0).([]byte)
 		}
 	}
 
-	var r1 int64
+	var r1 error
-	if rf, ok := ret.Get(1).(func(string) int64); ok {
+	if rf, ok := ret.Get(1).(func(string) error); ok {
 		r1 = rf(id)
-	} else {
-		r1 = ret.Get(1).(int64)
-	}
-
-	var r2 error
-	if rf, ok := ret.Get(2).(func(string) error); ok {
-		r2 = rf(id)
-	} else {
-		r2 = ret.Error(2)
-	}
-
-	return r0, r1, r2
-}
-
-// Save provides a mock function with given fields: fileName, userID, r
-func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) {
-	ret := _m.Called(fileName, userID, r)
-
-	var r0 string
-	if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok {
-		r0 = rf(fileName, userID, r)
-	} else {
-		r0 = ret.Get(0).(string)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok {
-		r1 = rf(fileName, userID, r)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -95,36 +83,15 @@ func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string,
 	return r0, r1
 }
 
-// SaveWithID provides a mock function with given fields: id, r
+// Save provides a mock function with given fields: id, img
-func (_m *MockStore) SaveWithID(id string, r io.Reader) (string, error) {
+func (_m *MockStore) Save(id string, img []byte) error {
-	ret := _m.Called(id, r)
+	ret := _m.Called(id, img)
 
-	var r0 string
+	var r0 error
-	if rf, ok := ret.Get(0).(func(string, io.Reader) string); ok {
+	if rf, ok := ret.Get(0).(func(string, []byte) error); ok {
-		r0 = rf(id, r)
+		r0 = rf(id, img)
 	} else {
-		r0 = ret.Get(0).(string)
+		r0 = ret.Error(0)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string, io.Reader) error); ok {
-		r1 = rf(id, r)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
-// SizeLimit provides a mock function with given fields:
-func (_m *MockStore) SizeLimit() int {
-	ret := _m.Called()
-
-	var r0 int
-	if rf, ok := ret.Get(0).(func() int); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(int)
 	}
 
 	return r0

backend/app/store/image/image_test.go

@@ -3,9 +3,14 @@ package image
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
+	"fmt"
 	"image"
+	"io"
 	"io/ioutil"
+	"os"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -14,8 +19,62 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestService_SaveAndLoad(t *testing.T) {
+	store := MockStore{}
+	svc := NewService(&store, ServiceParams{MaxSize: 1500, MaxWidth: 32, MaxHeight: 32})
+
+	store.On("Save", "test_id", mock.Anything).Return(nil)
+	err := svc.SaveWithID("test_id", gopherPNG())
+	assert.NoError(t, err)
+
+	store.On("Load", "test_id", mock.Anything).Return(nil, nil)
+	img, err := svc.Load("test_id")
+	assert.NoError(t, err)
+	assert.Nil(t, img)
+}
+
+func TestService_Resize(t *testing.T) {
+	img, err := readAndValidateImage(gopherPNG(), 1500)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+
+	img = resize(img, 32, 32)
+	assert.Equal(t, 1135, len(img))
+}
+
+func TestService_ResizeJpeg(t *testing.T) {
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+
+	img, err := readAndValidateImage(fh, 32000)
+	assert.NoError(t, err)
+	assert.Equal(t, 23983, len(img))
+
+	img = resize(img, 400, 300)
+	assert.Equal(t, 10918, len(img))
+}
+
+func TestService_SaveTooLarge(t *testing.T) {
+	svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}}
+	svc.MaxSize = 2000
+	_, err := svc.Save("user2", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+	err = svc.SaveWithID("test_id", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestService_WrongFormat(t *testing.T) {
+	svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}}
+
+	_, err := svc.Save("user1", strings.NewReader("blah blah bad image"))
+	assert.Error(t, err)
+}
+
 func TestService_ExtractPictures(t *testing.T) {
-	svc := Service{ImageAPI: "/blah/"}
+	svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/", ProxyAPI: "/non_existent"}}
 	html := `blah <img src="/blah/user1/pic1.png"/> foo
 <img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
 	ids, err := svc.ExtractPictures(html)
@@ -23,23 +82,51 @@ func TestService_ExtractPictures(t *testing.T) {
 	require.Equal(t, 2, len(ids), "two images")
 	assert.Equal(t, "user1/pic1.png", ids[0])
 	assert.Equal(t, "user2/pic3.png", ids[1])
-}
 
-func TestService_ExtractPictures2(t *testing.T) {
+	svc = Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/", ProxyAPI: "https://remark42.radio-t.com/api/v1/img"}}
-	svc := Service{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}
+	html = `<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде.
-	html := "<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." +
+</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля).
-		"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
+Например одна из моих вот так выглядит на libraries.io:</p>\n\n
-	ids, err := svc.ExtractPictures(html)
+<p><img src="https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png" alt="bjtr0-201906-08110846-i324c.png"/></p>\n\n<p>
+По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е.
+<code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n`
+	ids, err = svc.ExtractPictures(html)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
+
+	// proxied image
+	html = `<img src="https://remark42.radio-t.com/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(ids), "one image in")
+	assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
+
+	// bad url
+	html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
+
+	// bad src
+	html = `<img src="https://remark42.radio-t.com/api/v1/img?src=bad">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
+
+	// good src with bad content
+	badURL := base64.URLEncoding.EncodeToString([]byte(" http://foo.bar"))
+	html = fmt.Sprintf(`<img src="https://remark42.radio-t.com/api/v1/img?src=%s">`, badURL)
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
 }
 
 func TestService_Cleanup(t *testing.T) {
 	store := MockStore{}
 	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
 
-	svc := Service{Store: &store, TTL: 100 * time.Millisecond}
+	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
 	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
 	defer cancel()
 	svc.Cleanup(ctx)
@@ -49,51 +136,62 @@ func TestService_Cleanup(t *testing.T) {
 func TestService_Submit(t *testing.T) {
 	store := MockStore{}
 	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
-	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}}
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	store.AssertNumberOfCalls(t, "Commit", 0)
 	time.Sleep(time.Millisecond * 150)
 	store.AssertNumberOfCalls(t, "Commit", 5)
+	svc.Close(context.TODO())
 }
 
 func TestService_Close(t *testing.T) {
 	store := MockStore{}
 	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
-	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
-	svc.Close()
+	svc.Close(context.TODO())
 	store.AssertNumberOfCalls(t, "Commit", 5)
 }
 
 func TestService_SubmitDelay(t *testing.T) {
 	store := MockStore{}
 	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
-	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
 	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	store.AssertNumberOfCalls(t, "Commit", 3)
-	svc.Close()
+	svc.Close(context.TODO())
 	store.AssertNumberOfCalls(t, "Commit", 5)
 }
 
-func TestService_resize(t *testing.T) {
+func TestService_Info(t *testing.T) {
+	store := MockStore{}
+	store.On("Info", mock.Anything, mock.Anything).Once().Return(StoreInfo{}, nil)
 
-	// Reader is nil.
+	svc := Service{store: &store, ServiceParams: ServiceParams{}}
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+	store.AssertNumberOfCalls(t, "Info", 1)
+}
+
+func TestService_resize(t *testing.T) {
+	// reader is nil
 	resized := resize(nil, 100, 100)
 	assert.Nil(t, resized)
 
-	// Negative limit error.
+	// negative limit error
 	resized = resize([]byte("some picture bin data"), -1, -1)
 	require.NotNil(t, resized)
 	assert.Equal(t, resized, []byte("some picture bin data"))
 
-	// Decode error.
+	// decode error
 	resized = resize([]byte("invalid image content"), 100, 100)
 	assert.NotNil(t, resized)
 	assert.Equal(t, resized, []byte("invalid image content"))
@@ -110,15 +208,14 @@ func TestService_resize(t *testing.T) {
 		img, err := ioutil.ReadFile(c.file)
 		require.NoError(t, err, "can't open test file %s", c.file)
 
-		// No need for resize, image dimensions are smaller than resize limit.
+		// no need for resize, image dimensions are smaller than resize limit
 		resized = resize(img, 800, 800)
 		assert.NotNil(t, resized, "file %s", c.file)
 		assert.Equal(t, resized, img)
 
-		// Resizing to half of width. Check resized image format PNG.
+		// resizing to half of width
 		resized = resize(img, 400, 400)
 		assert.NotNil(t, resized, "file %s", c.file)
-
 		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
 		assert.NoError(t, err, "file %s", c.file)
 		assert.Equal(t, "png", format, "file %s", c.file)
@@ -142,6 +239,7 @@ func TestGetProportionalSizes(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			resW, resH := getProportionalSizes(tt.inpW, tt.inpH, tt.limitW, tt.limitH)
 			assert.Equal(t, tt.resW, resW, "width")
@@ -149,3 +247,13 @@ func TestGetProportionalSizes(t *testing.T) {
 		})
 	}
 }
+
+func TestCachedImgID(t *testing.T) {
+	img, err := CachedImgID(" http://foo.com")
+	assert.Error(t, err)
+	assert.Empty(t, img)
+	imgURL := "http://example.org/img/1.png"
+	img, err = CachedImgID(imgURL)
+	assert.NoError(t, err)
+	assert.Equal(t, "cached_images/"+Sha1Str("example.org")+"-"+Sha1Str(imgURL), img)
+}

backend/app/store/image/remote_store.go

@@ -0,0 +1,61 @@
+package image
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	jrpc.Client
+}
+
+// Save saves image with given id to staging.
+func (r *RPC) Save(id string, img []byte) error {
+	_, err := r.Call("image.save_with_id", id, img)
+	return err
+}
+
+// Load image with given id
+func (r *RPC) Load(id string) ([]byte, error) {
+	resp, err := r.Call("image.load", id)
+	if err != nil {
+		return nil, err
+	}
+	var rawImg string
+	if err = json.Unmarshal(*resp.Result, &rawImg); err != nil {
+		return nil, err
+	}
+	return ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg)))
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (r *RPC) Commit(id string) error {
+	_, err := r.Call("image.commit", id)
+	return err
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (r *RPC) Cleanup(_ context.Context, ttl time.Duration) error {
+	_, err := r.Call("image.cleanup", ttl)
+	return err
+}
+
+// Info returns meta information about storage
+func (r *RPC) Info() (StoreInfo, error) {
+	resp, err := r.Call("image.info")
+	if err != nil {
+		return StoreInfo{}, err
+	}
+	info := StoreInfo{}
+	if err = json.Unmarshal(*resp.Result, &info); err != nil {
+		return StoreInfo{}, err
+	}
+	return info, err
+}

backend/app/store/image/remote_store_test.go

@@ -0,0 +1,89 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemote_SaveWithID(t *testing.T) {
+	ts := testServer(t, fmt.Sprintf(`{"method":"image.save_with_id","params":["54321","%s"],"id":1}`, gopher),
+		`{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.Save("54321", gopherPNGBytes())
+	assert.NoError(t, err)
+}
+
+func TestRemote_Load(t *testing.T) {
+	ts := testServer(t, `{"method":"image.load","params":"54321","id":1}`,
+		fmt.Sprintf(`{"result":"%v","id":1}`, gopher))
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Load("54321")
+	assert.NoError(t, err)
+	assert.Equal(t, gopherPNGBytes(), res)
+}
+
+func TestRemote_Commit(t *testing.T) {
+	ts := testServer(t, `{"method":"image.commit","params":"gopher_id","id":1}`, `{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.Commit("gopher_id")
+	assert.NoError(t, err)
+}
+
+func TestRemote_Cleanup(t *testing.T) {
+	ts := testServer(t, `{"method":"image.cleanup","params":60000000000,"id":1}`, `{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestRemote_Info(t *testing.T) {
+	ts := testServer(t, `{"method":"image.info","id":1}`,
+		`{"result":{"FirstStagingImageTS":"0001-01-01T00:00:01Z"},"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	info, err := c.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		_, _ = fmt.Fprint(w, resp)
+	}))
+}

backend/app/store/service/restricted_words.go

@@ -19,7 +19,7 @@ type StaticRestrictedWordsLister struct {
 }
 
 // List provides restricted words in comments (ignores siteID)
-func (l StaticRestrictedWordsLister) List(siteID string) (restricted []string, err error) {
+func (l StaticRestrictedWordsLister) List(_ string) (restricted []string, err error) {
 	return l.Words, nil
 }
 

backend/app/store/service/service.go

@@ -1,6 +1,5 @@
 // Package service wraps engine interfaces with common logic unrelated to any particular engine implementation.
 // All consumers should be using service.DataStore and not the naked engine!
-
 package service
 
 import (
@@ -102,7 +101,7 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
-	s.submitImages(comment)
+	s.submitImages(comment.Locator, comment.ID)
 	if e := s.AdminStore.OnEvent(comment.Locator.SiteID, admin.EvCreate); e != nil {
 		log.Printf("[WARN] failed to send create event, %s", e)
 	}
@@ -201,24 +200,46 @@ func (s *DataStore) DeleteUserDetail(siteID string, userID string, detail engine
 	})
 }
 
-// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+// ResubmitStagingImages retrieves timestamp of the oldest image in staging and
-func (s *DataStore) submitImages(comment store.Comment) {
+// calls s.submitImages on all comments newer than it
+func (s *DataStore) ResubmitStagingImages(sites []string) error {
+	info, err := s.ImageService.Info()
+	if err != nil {
+		return err
+	}
+	ts := info.FirstStagingImageTS
+	if ts.IsZero() {
+		return nil
+	}
+	result := new(multierror.Error)
+	for _, site := range sites {
+		locator := store.Locator{SiteID: site}
+		comments, err := s.FindSince(locator, "time", store.User{}, ts)
+		result = multierror.Append(result, errors.Wrapf(err, "problem finding comments for site %s", site))
+		for _, c := range comments {
+			s.submitImages(c.Locator, c.ID)
+		}
+	}
+	return result.ErrorOrNil()
+}
 
-	s.ImageService.Submit(func() []string {
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
-		c := comment
+func (s *DataStore) submitImages(locator store.Locator, commentID string) {
+
+	s.ImageService.Submit(func() []string { // get all ids from comment's text
 		// this can be called after last edit, we have to retrieve fresh comment
-		cc, err := s.Engine.Get(engine.GetRequest{Locator: c.Locator, CommentID: c.ID})
+		cc, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 		if err != nil {
-			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", commentID, err)
 			return nil
 		}
 		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
 		if err != nil {
-			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			log.Printf("[WARN] can't get extract pictures from %s, %v", commentID, err)
 			return nil
 		}
 		if len(imgIds) > 0 {
-			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", commentID, imgIds)
 		}
 		return imgIds
 	})
@@ -881,6 +902,7 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co
 	}
 
 	c = s.prepVotes(c, user)
+	c.Locator.URL = c.SanitizeAsURL(c.Locator.URL) // urls prior to #927
 	return c
 }
 

backend/app/store/service/service_test.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -15,12 +16,11 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -99,7 +99,6 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-
 	ks := admin.NewStaticKeyStore("secret 123")
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
@@ -572,6 +571,7 @@ func TestService_Controversy(t *testing.T) {
 
 	b := DataStore{}
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(fmt.Sprintf("check-%d-%d:%d", i, tt.ups, tt.downs), func(t *testing.T) {
 			assert.InDelta(t, tt.res, b.controversy(tt.ups, tt.downs), 0.01)
 		})
@@ -1277,8 +1277,15 @@ func TestService_submitImages(t *testing.T) {
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
 	mockStore := image.MockStore{}
-	mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil)
+	mockStore.On("Commit", "dev/pic1.png").Once().Return(nil)
-	imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
+	mockStore.On("Commit", "dev/pic2.png").Once().Return(nil)
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 50 * time.Millisecond,
+			ImageAPI:     "/",
+			ProxyAPI:     "/non_existent",
+		})
+	defer imgSvc.Close(context.TODO())
 
 	// two comments for https://radio-t.com
 	eng, teardown := prepStoreEngine(t)
@@ -1296,8 +1303,111 @@ func TestService_submitImages(t *testing.T) {
 	_, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
 	assert.NoError(t, err)
 
-	b.submitImages(c)
+	b.submitImages(c.Locator, c.ID)
 	time.Sleep(250 * time.Millisecond)
+	mockStore.AssertNumberOfCalls(t, "Commit", 2)
+}
+
+func TestService_ResubmitStagingImages(t *testing.T) {
+	mockStore := image.MockStore{}
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 10 * time.Millisecond,
+			ImageAPI:     "http://127.0.0.1:8080/api/v1/picture/",
+			ProxyAPI:     "http://127.0.0.1:8080/api/v1/img",
+		})
+	defer imgSvc.Close(context.TODO())
+
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, EditDuration: 10 * time.Millisecond, ImageService: imgSvc}
+
+	// create comment with three images without preparing it properly
+	comment := store.Comment{
+		ID: "id-0",
+		Text: `<img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf122eq9r8ad657n3ng" alt="startrails_01.jpg"><br/>
+               <img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf321eq9r8ad657n3ng" alt="cat.png"><br/>
+               <img src="http://127.0.0.1:8080/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png"><br/>
+               <img src="https://homepages.cae.wisc.edu/~ece533/images/boat.png" alt="boat.png">`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Engine.Create(comment)
+	require.NoError(t, err)
+
+	// resubmit single comment with three images, of which two are in staging storage
+	mockStore.On("Info").Once().Return(image.StoreInfo{FirstStagingImageTS: time.Time{}.Add(time.Second)}, nil)
+	err = b.ResubmitStagingImages([]string{"radio-t"})
+	assert.NoError(t, err)
+
+	// wait for Submit goroutine to commit image
+	mockStore.On("Commit", "dev_user/bqf122eq9r8ad657n3ng").Once().Return(nil)
+	mockStore.On("Commit", "dev_user/bqf321eq9r8ad657n3ng").Once().Return(nil)
+	mockStore.On("Commit", "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e").Once().Return(nil)
+	time.Sleep(time.Millisecond * 100)
+
+	mockStore.AssertNumberOfCalls(t, "Info", 1)
+	mockStore.AssertNumberOfCalls(t, "Commit", 3)
+
+	// empty answer
+	mockStoreEmpty := image.MockStore{}
+	imgSvcEmpty := image.NewService(&mockStoreEmpty,
+		image.ServiceParams{
+			EditDuration: 10 * time.Millisecond,
+			ImageAPI:     "http://127.0.0.1:8080/api/v1/picture/",
+		})
+	defer imgSvcEmpty.Close(context.TODO())
+	bEmpty := DataStore{Engine: eng, EditDuration: 10 * time.Millisecond, ImageService: imgSvcEmpty}
+
+	// resubmit receive empty timestamp and should do nothing
+	mockStoreEmpty.On("Info").Once().Return(image.StoreInfo{FirstStagingImageTS: time.Time{}}, nil)
+	err = bEmpty.ResubmitStagingImages([]string{"radio-t", "non_existent"})
+	assert.NoError(t, err)
+
+	mockStoreEmpty.AssertNumberOfCalls(t, "Info", 1)
+
+	// 	error from image storage
+	mockStoreError := image.MockStore{}
+	imgSvcError := image.NewService(&mockStoreError,
+		image.ServiceParams{
+			EditDuration: 10 * time.Millisecond,
+			ImageAPI:     "http://127.0.0.1:8080/api/v1/picture/",
+		})
+	defer imgSvcError.Close(context.TODO())
+	bError := DataStore{Engine: eng, EditDuration: 10 * time.Millisecond, ImageService: imgSvcError}
+
+	// resubmit will receive error from image storage and should return it
+	mockStoreError.On("Info").Once().Return(image.StoreInfo{}, errors.New("mock_err"))
+	err = bError.ResubmitStagingImages([]string{"radio-t"})
+	assert.EqualError(t, err, "mock_err")
+
+	mockStoreError.AssertNumberOfCalls(t, "Info", 1)
+}
+
+func TestService_ResubmitStagingImages_EngineError(t *testing.T) {
+	mockStore := image.MockStore{}
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 10 * time.Millisecond,
+			ImageAPI:     "http://127.0.0.1:8080/api/v1/picture/",
+		})
+	defer imgSvc.Close(context.TODO())
+
+	engineMock := engine.MockInterface{}
+	site1Req := engine.FindRequest{Locator: store.Locator{SiteID: "site1", URL: ""}, Sort: "time", Since: time.Time{}.Add(time.Second)}
+	site2Req := engine.FindRequest{Locator: store.Locator{SiteID: "site2", URL: ""}, Sort: "time", Since: time.Time{}.Add(time.Second)}
+	engineMock.On("Find", site1Req).Return(nil, nil)
+	engineMock.On("Find", site2Req).Return(nil, errors.New("mockError"))
+	b := DataStore{Engine: &engineMock, EditDuration: 10 * time.Millisecond, ImageService: imgSvc}
+
+	// One call without error and one with error
+	mockStore.On("Info").Once().Return(image.StoreInfo{FirstStagingImageTS: time.Time{}.Add(time.Second)}, nil)
+	err := b.ResubmitStagingImages([]string{"site1", "site2"})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "problem finding comments for site site2: mockError")
+
+	mockStore.AssertNumberOfCalls(t, "Info", 1)
 }
 
 func TestService_alterComment(t *testing.T) {

backend/app/store/service/title_test.go

@@ -31,6 +31,7 @@ func TestTitle_GetTitle(t *testing.T) {
 
 	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(fmt.Sprintf("check-%d", i), func(t *testing.T) {
 			title, ok := ex.getTitle(strings.NewReader(tt.page))
 			assert.Equal(t, tt.ok, ok)

backend/app/store/service/tree_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -84,7 +83,7 @@ func TestMakeEmptySubtree(t *testing.T) {
 	res := MakeTree(comments, "time", 0)
 	resJSON, err := json.Marshal(&res)
 	require.NoError(t, err)
-	log.Print(string(resJSON))
+	t.Log(string(resJSON))
 
 	expJSON := mustLoadJSONFile(t, "testdata/tree_del.json")
 	assert.Equal(t, string(expJSON), string(resJSON))

backend/app/store/user_test.go

@@ -60,8 +60,8 @@ func TestUser_HashFailed(t *testing.T) {
 
 type mockHash struct{}
 
-func (mock mockHash) Sum(b []byte) []byte               { return nil }
+func (mock mockHash) Sum(_ []byte) []byte               { return nil }
 func (mock mockHash) Reset()                            {}
 func (mock mockHash) Size() int                         { return 0 }
 func (mock mockHash) BlockSize() int                    { return 0 }
-func (mock mockHash) Write(p []byte) (n int, err error) { return 0, errors.New("error") }
+func (mock mockHash) Write(_ []byte) (n int, err error) { return 0, errors.New("error") }

backend/go.mod

@@ -1,66 +1,38 @@
 module github.com/umputun/remark/backend
 
-go 1.13
+go 1.14
-
-replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1
 
 require (
-	cloud.google.com/go v0.47.0 // indirect
+	github.com/Depado/bfchroma v1.2.0
-	github.com/PuerkitoBio/goquery v1.5.0
+	github.com/PuerkitoBio/goquery v1.5.1
-	github.com/ajg/form v1.5.1 // indirect
+	github.com/alecthomas/chroma v0.7.2
-	github.com/andybalholm/cascadia v1.1.0 // indirect
-	github.com/coreos/bbolt v1.3.3
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/didip/tollbooth v4.0.2+incompatible
 	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
-	github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 // indirect
+	github.com/go-chi/chi v4.1.0+incompatible
-	github.com/gavv/httpexpect v2.0.0+incompatible // indirect
+	github.com/go-chi/cors v1.1.1
-	github.com/go-chi/chi v4.0.2+incompatible
-	github.com/go-chi/cors v1.0.0
 	github.com/go-chi/render v1.0.1
-	github.com/go-pkgz/auth v0.9.0
+	github.com/go-pkgz/auth v0.10.1
 	github.com/go-pkgz/jrpc v0.1.0
 	github.com/go-pkgz/lcw v0.5.0
-	github.com/go-pkgz/lgr v0.6.3
+	github.com/go-pkgz/lgr v0.7.0
 	github.com/go-pkgz/repeater v1.1.3
-	github.com/go-pkgz/rest v1.4.1
+	github.com/go-pkgz/rest v1.5.0
 	github.com/go-pkgz/syncs v1.1.1
 	github.com/google/uuid v1.1.1
-	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/gorilla/feeds v1.1.1
-	github.com/gorilla/websocket v1.4.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0
-	github.com/hashicorp/go-multierror v1.0.0
+	github.com/kyokomi/emoji v2.2.1+incompatible
-	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
-	github.com/klauspost/compress v1.7.6 // indirect
-	github.com/klauspost/cpuid v1.2.1 // indirect
-	github.com/kyokomi/emoji v2.1.0+incompatible
-	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/mattn/go-isatty v0.0.9 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.2
-	github.com/onsi/ginkgo v1.9.0 // indirect
-	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
-	github.com/rakyll/statik v0.1.6
+	github.com/rakyll/statik v0.1.7
 	github.com/rs/xid v1.2.1
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1
-	github.com/smartystreets/assertions v1.0.1 // indirect
+	github.com/stretchr/testify v1.5.1
-	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/umputun/go-flags v1.5.1
-	github.com/stretchr/testify v1.4.0
+	go.etcd.io/bbolt v1.3.4
-	github.com/tidwall/buntdb v1.1.0 // indirect
+	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
-	github.com/tidwall/gjson v1.3.2 // indirect
+	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	github.com/valyala/fasthttp v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	github.com/xeipuuv/gojsonschema v1.1.0 // indirect
-	go.mongodb.org/mongo-driver v1.1.2 // indirect
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
-	golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
-	golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 // indirect
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	google.golang.org/appengine v1.6.5 // indirect
-	gopkg.in/oauth2.v3 v3.11.0 // indirect
-	gopkg.in/russross/blackfriday.v2 v2.0.1
-	gopkg.in/yaml.v2 v2.2.4 // indirect
 )

backend/go.sum

@@ -2,39 +2,39 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
 cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.47.0 h1:1JUtpcY9E7+eTospEwWS2QXP3DEn7poB3E2j0jN74mM=
-cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
-github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
-github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
-github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
-github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.6.0 h1:gcvXlpe0/NoQP3BvneRfgcauLIJDw9VblkoFwZ5XGFs=
+github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
+github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
+github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
-github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
-github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
-github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -46,35 +46,38 @@ github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7z
 github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
 github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
 github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
-github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 h1:DddqAaWDpywytcG8w/qoQ5sAN8X12d3Z3koB0C3Rxsc=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
-github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313 h1:GSPjYG49Uqn3S1oeFgJtlGI3ykTavl/yvYgZlz6wsoI=
 github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
-github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8=
+github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d h1:oYXrtNhqNKL1dVtKdv8XUq5zqdGVFNQ0/4tvccXZOLM=
-github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
 github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/chi v4.1.0+incompatible h1:ETj3cggsVIY2Xao5ExCu6YhEh5MD6JTfcBzS37R260w=
-github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1 h1:eHuqxsIw89iXcWnWUN8R72JMibABJTN/4IOYI5WERvw=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-pkgz/auth v0.10.1 h1:GYf64js5n/oVEPXhRShyDDiR/oahdZOMLbxmnnGX2IU=
-github.com/go-pkgz/auth v0.9.0 h1:1BR+wtbkHzLg4BH1CQUpEO/WzgihrDqzW8XN4g9Pq9g=
+github.com/go-pkgz/auth v0.10.1/go.mod h1:wxyQqc0UUP1jT4l6zk1r6XPcVdcgIzW2OiQ8hBEHd64=
-github.com/go-pkgz/auth v0.9.0/go.mod h1:8Kiold3xA9VUiYZwMyMhwBgiGImZFoL/EmtO9lSIQxo=
 github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
 github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
 github.com/go-pkgz/lcw v0.5.0 h1:dSf98y5067zRmBA6fo+gbqLGLy08RGUHoaecwtjDyOE=
 github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
-github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
-github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
 github.com/go-pkgz/repeater v1.1.3 h1:q6+JQF14ESSy28Dd7F+wRelY4F+41HJ0LEy/szNnMiE=
 github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
 github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
 github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
 github.com/go-pkgz/syncs v1.1.1 h1:jWN+y6FS/Xe+8z4l3QMbSnODGyaxDHGojIS+wyKIjxg=
 github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
 github.com/go-redis/redis/v7 v7.0.0-beta.4 h1:p6z7Pde69EGRWvlC++y8aFcaWegyrKHzOBGo0zUACTQ=
@@ -86,7 +89,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
@@ -98,7 +100,6 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -106,24 +107,19 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
 github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
@@ -134,60 +130,51 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
-github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.7.6 h1:GH2karLOcuZtA5a3+KuzSU33A2cvcHGbtEWM6K4t7oU=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM=
-github.com/klauspost/compress v1.7.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kyokomi/emoji v2.1.0+incompatible h1:+DYU2RgpI6OHG4oQkM5KlqD3Wd3UPEsX8jamTo1Mp6o=
+github.com/kyokomi/emoji v2.2.1+incompatible h1:uP/6J5y5U0XxPh6fv8YximpVD1uMrshXG78I1+uF5SA=
-github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
-github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
-github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
+github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
-github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -196,28 +183,23 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
-github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0 h1:QnyrPZZvPmR0AtJCxxfCtI1qN+fYpKTKJ/5opWmZ34k=
 github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.0.0 h1:urIJqQ8OR9fibXXtFSu8sR5arMqZK8ZnNq22yWl+A+8=
 github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
-github.com/tidwall/buntdb v1.1.0 h1:H6LzK59KiNjf1nHVPFrYj4Qnl8d8YLBsYamdL8N+Bao=
+github.com/tidwall/gjson v1.1.3 h1:u4mspaByxY+Qk4U1QYYVzGFI8qxN/3jtEV0ZDb2vRic=
-github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
 github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
-github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
-github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
@@ -228,24 +210,23 @@ github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2K
 github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.0.0 h1:BwIoZQbBsTo3v2F5lz5Oy3TlTq4wLKTLV260EVTEWco=
 github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/fasthttp v1.4.0 h1:PuaTGZIw3mjYhhhbVbCQp8aciRZN9YdoB7MGX9Ko76A=
-github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b h1:6cLsL+2FW6dRAdl5iMtHgRogVCff0QpRi9653YmdcJA=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50=
 github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
-github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
@@ -256,63 +237,45 @@ github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcm
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.mongodb.org/mongo-driver v1.1.1 h1:Sq1fR+0c58RME5EoqKdjkiQAmPjmfHlZOoRI6fTUOcs=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA=
-go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff h1:+2zgJKVDVAz/BWSsuniCmU1kLCjL88Z8/kv39xCI9NQ=
 golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a h1:Yu34BogBivvmu7SAzHHaB9nZWH5D1C+z3F1jyIaYZSQ=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -324,25 +287,19 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -351,70 +308,41 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/oauth2.v3 v3.10.1 h1:/abis3O6tZFizY1/FgKGoDOmmu2ddvscBSSPdHVa6OI=
 gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
-gopkg.in/oauth2.v3 v3.11.0 h1:ECNylTd20FuK06WQ6jfpvtKlmHPkqQQw90Lc/HxZoD0=
-gopkg.in/oauth2.v3 v3.11.0/go.mod h1:ZdKwOng79bFq17t/ItHGtL8LJ2dW3tmrdJJhHmqS+uc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/vendor/cloud.google.com/go/compute/metadata/.repo-metadata.json

@@ -1,12 +0,0 @@
-{
-  "name": "metadata",
-  "name_pretty": "Google Compute Engine Metadata API",
-  "product_documentation": "https://cloud.google.com/compute/docs/storing-retrieving-metadata",
-  "client_documentation": "https://godoc.org/cloud.google.com/go/compute/metadata",
-  "release_level": "ga",
-  "language": "go",
-  "repo": "googleapis/google-cloud-go",
-  "distribution_name": "cloud.google.com/go/compute/metadata",
-  "api_id": "compute:metadata",
-  "requires_billing": false
-}

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -227,9 +227,6 @@ func InternalIP() (string, error) { return defaultClient.InternalIP() }
 // ExternalIP returns the instance's primary external (public) IP address.
 func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
 
-// Email calls Client.Email on the default client.
-func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) }
-
 // Hostname returns the instance's hostname. This will be of the form
 // "<instanceID>.c.<projID>.internal".
 func Hostname() (string, error) { return defaultClient.Hostname() }
@@ -370,16 +367,6 @@ func (c *Client) InternalIP() (string, error) {
 	return c.getTrimmed("instance/network-interfaces/0/ip")
 }
 
-// Email returns the email address associated with the service account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
-func (c *Client) Email(serviceAccount string) (string, error) {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email")
-}
-
 // ExternalIP returns the instance's primary external (public) IP address.
 func (c *Client) ExternalIP() (string, error) {
 	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")