move back css for last comments

with zh.json and en.json.

.dockerignore

@@ -21,4 +21,4 @@ debug.test
 *.test
 remark42
 /backend/var/
-compose-private-backend.yml
+compose-private-backend.yml

.drone.yml

@@ -85,6 +85,8 @@ steps:
     commands:
       - ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
       - ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
+      - ssh umputun@remark42.com "cd /srv/remark-site && git pull && git submodule update --recursive --remote"
+      - ssh umputun@remark42.com "cd /srv/remark-site && docker-compose build && docker-compose up"
     when:
       branch: master
       event: push

.github/FUNDING.yml

@@ -1,4 +1,4 @@
 # These are supported funding model platforms
 
 github: [umputun]
-#patreon: remark42
+patreon: remark42

.github/workflows/ci-frontend-size-limit.yml

@@ -0,0 +1,19 @@
+name: frontend
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/ci-frontend-size-limit.yml'
+      - 'frontend/**'
+
+jobs:
+  size:
+    runs-on: ubuntu-latest
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - uses: actions/checkout@v1
+      - uses: andresz1/size-limit-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          directory: frontend

.github/workflows/ci-frontend.yml

@@ -0,0 +1,105 @@
+name: frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-frontend.yml'
+      - 'frontend/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-frontend.yml'
+      - 'frontend/**'
+
+jobs:
+  check-transtations:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:translation
+        working-directory: ./frontend
+
+  check-typescript:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:types
+        working-directory: ./frontend
+
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npx run-p lint
+        working-directory: ./frontend
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npm run test:coverage
+        working-directory: ./frontend
+
+      - name: submit coverage
+        run: node ${{ github.workspace }}/frontend/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-test-backend.yml

@@ -8,13 +8,11 @@ on:
       - '.github/workflows/ci-test-backend.yml'
       - 'backend/**'
       - '!backend/scripts/**'
-      - '.golangci.yml'
   pull_request:
     paths:
       - '.github/workflows/ci-test-backend.yml'
       - 'backend/**'
       - '!backend/scripts/**'
-      - '.golangci.yml'
 
 jobs:
   backend:
@@ -24,41 +22,39 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: debug if needed
-        run: |
-          export DEBUG=${DEBUG:-false}
-          if [[ "$DEBUG" == "true" ]]; then
-            env
-          fi
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
         env:
           DEBUG: ${{secrets.DEBUG}}
 
       - name: install go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.13
-
-      - name: test backend
-        run: |
-          export TZ="America/Chicago"
-          date
-          cd backend/app
-          go test -mod=vendor -timeout=60s -covermode=count -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
-          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
-          cd ../_example/memory_store
-          go test -race ./...
+          go-version: 1.14
 
       - name: install golangci-lint and goveralls
         run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.20.0
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.26.0
           go get -u github.com/mattn/goveralls
 
-      - name: run backend linters
-        run: $GITHUB_WORKSPACE/golangci-lint run --config .golangci.yml ./...
-        working-directory: backend
+      - name: test and lint backend
+        run: |
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          GOFLAGS: "-mod=vendor"
+          TZ: "America/Chicago"
 
-      - name: run linters for examples
-        run: $GITHUB_WORKSPACE/golangci-lint run --config ${GITHUB_WORKSPACE}/backend/.golangci.yml ./...
+      - name: test and lint examples
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
         working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
 
       - name: submit coverage
         run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov

.github/workflows/ci-test-frontend.yml

@@ -1,28 +0,0 @@
-name: test_frontend
-
-on:
-  push:
-    branches:
-    tags:
-    paths:
-      - '.github/workflows/ci-test-frontend.yml'
-      - 'frontend/**'
-  pull_request:
-    paths:
-      - '.github/workflows/ci-test-frontend.yml'
-      - 'frontend/**'
-
-jobs:
-  frontend:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/setup-node@v1
-
-      - run: npm ci
-        working-directory: ./frontend
-
-      - run: npx run-p check lint test
-        working-directory: ./frontend

.gitignore

@@ -19,6 +19,7 @@ debug.test
 remark42
 /bin/
 /backend/var/
+/backend/app/var/
 compose-private-backend.yml
 compose-private-frontend.yml
 compose-private.yml

.golangci.yml

@@ -1,2 +0,0 @@
-service:
-  golangci-lint-version: 1.19.1

Dockerfile

@@ -1,4 +1,4 @@
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM umputun/baseimage:buildgo-v1.6.1 as build-backend
 
 ARG CI
 ARG DRONE
@@ -16,11 +16,14 @@ WORKDIR /build/backend
 
 ENV GOFLAGS="-mod=vendor"
 
+# install gcc in order to be able to go test package with -race
+RUN apk --no-cache add gcc libc-dev
+
 # run tests
 RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
         golangci-lint run --config ../.golangci.yml ./... ; \
     else echo "skip backend tests and linter" ; fi
@@ -32,7 +35,7 @@ RUN \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM node:10.11-alpine as build-frontend-deps
+FROM node:12.16-alpine as build-frontend-deps
 
 ARG CI
 ENV HUSKY_SKIP_INSTALL=true
@@ -40,9 +43,9 @@ ENV HUSKY_SKIP_INSTALL=true
 RUN apk add --no-cache --update git
 ADD frontend/package.json /srv/frontend/package.json
 ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/frontend && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci --loglevel warn
 
-FROM node:10.11-alpine as build-frontend
+FROM node:12.16-alpine as build-frontend
 
 ARG CI
 ARG SKIP_FRONTEND_TEST
@@ -51,11 +54,11 @@ ARG NODE_ENV=production
 COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
 ADD frontend /srv/frontend
 RUN cd /srv/frontend && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p check lint test build ; \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-FROM umputun/baseimage:app
+FROM umputun/baseimage:app-v1.6.1
 
 WORKDIR /srv
 
@@ -66,6 +69,7 @@ ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
 COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-backend /build/backend/templates /srv
 COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42

Dockerfile.artifacts

@@ -1,4 +1,4 @@
-FROM node:10.11-alpine as build-frontend-deps
+FROM node:12.16-alpine as build-frontend-deps
 
 ARG CI
 ARG DRONE
@@ -13,7 +13,7 @@ ADD frontend/package.json /srv/frontend/package.json
 ADD frontend/package-lock.json /srv/frontend/package-lock.json
 RUN cd /srv/frontend && CI=true npm ci
 
-FROM node:10.11-alpine as build-frontend
+FROM node:12.16-alpine as build-frontend
 
 ARG CI
 ARG NODE_ENV=production
@@ -31,8 +31,6 @@ FROM umputun/baseimage:buildgo-latest as build-backend
 ARG GITHUB_TOKEN
 ENV SKIP_BACKEND_TEST=true
 
-RUN go get github.com/rakyll/statik
-
 WORKDIR /build/backend
 ADD backend /build/backend
 ADD README.md /build/
@@ -44,9 +42,10 @@ COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
     export WEB_ROOT=/build/backend/web && \
-    sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
-    sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
+    find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \
     statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
+    statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \
+    ls -la /build/backend/app/templates/statik.go && \
     ls -la /build/backend/app/rest/api/statik.go && \
     ls -la /build/backend/web/
 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2020 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -34,6 +34,7 @@ frontend:
 	docker-compose -f compose-dev-frontend.yml build
 
 rundev:
+	docker pull umputun/baseimage:buildgo-latest
 	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
 	docker-compose -f compose-private.yml up
 

README.md

@@ -1,10 +1,11 @@
-# remark42 [![Build Status](https://github.com/umputun/remark/workflows/build/badge.svg)](https://github.com/umputun/remark/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://codecov.io/gh/umputun/remark42)
+
 
 
 
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
-* Social login via Google, Twitter, Facebook, GitHub and Yandex
+* Social login via Google, Twitter, Facebook, Microsoft, GitHub and Yandex
 * Login via email
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
@@ -31,9 +32,9 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 <details><summary>Screenshots</summary>
 
 Comments example:
-![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
+![](https://github.com/umputun/remark42/blob/master/screenshots/comments.png)
 
-For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark42/wiki/Admin-UI)
 </details>
 
 
@@ -42,7 +43,7 @@ For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki
   - [Install](#install)
     - [Backend](#backend)
       - [With Docker](#with-docker)
-      - [Without docker](#without-docker)
+      - [Without Docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
       - [Quick installation test](#quick-installation-test)
@@ -89,15 +90,15 @@ _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
 * make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
-* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
+* pull prepared images from the DockerHub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
-#### Without docker
+#### Without Docker
 
-* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
+* download archive for [stable release](https://github.com/umputun/remark42/releases) or [development version](https://remark42.com/downloads)
 * unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
 * run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
-* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
+* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
 
 #### Parameters
 
@@ -109,10 +110,12 @@ _this is the recommended way to run remark42_
 | store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `rpc`                |
 | store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                          |
 | store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                           |
-| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_         |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                     |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin ids (list of user ids), _multi_           |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin emails, _multi_                           |
 | backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                |
 | max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                        |
+| cache.type              | CACHE_TYPE              | `mem`                    | type of cache, `redis_pub_sub` or `mem` or `none` |
+| cache.redis_addr        | CACHE_REDIS_ADDR        | `127.0.0.1:6379`         | address of redis PubSub instance, turn `redis_pub_sub` cache on for distributed cache |
 | cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited     |
 | cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited       |
 | cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited  |
@@ -131,12 +134,16 @@ _this is the recommended way to run remark42_
 | image.resize-height     | IMAGE_RESIZE_HEIGHT     | `900`                    | height of resized image                         |
 | auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                         |
 | auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                      |
+| auth.send-jwt-header    | AUTH_SEND_JWT_HEADER    | `false`                  | send JWT as a header instead of cookie          |
+| auth.same-site          | AUTH_SAME_SITE          | `default`                | set same site policy for cookies (`default`, `none`, `lax` or `strict`)|
 | auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                          |
 | auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                      |
 | auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                        |
 | auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                    |
-| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                          |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                      |
+| auth.microsoft.cid      | AUTH_MICROSOFT_CID      |                          | Microsoft OAuth client ID                       |
+| auth.microsoft.csec     | AUTH_MICROSOFT_CSEC     |                          | Microsoft OAuth client secret                   |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | GitHub OAuth client ID                          |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | GitHub OAuth client secret                      |
 | auth.twitter.cid        | AUTH_TWITTER_CID        |                          | Twitter Consumer API Key                        |
 | auth.twitter.csec       | AUTH_TWITTER_CSEC       |                          | Twitter Consumer API Secret key                 |
 | auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                          |
@@ -155,6 +162,7 @@ _this is the recommended way to run remark42_
 | notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
 | notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
 | notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
+| notify.email.notify_admin | NOTIFY_EMAIL_ADMIN    | `false`                  | notify admin on new comments via ADMIN_SHARED_EMAIL |
 | smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
 | smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
 | smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
@@ -176,12 +184,15 @@ _this is the recommended way to run remark42_
 | critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                        |
 | positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive   |
 | restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_ |
+| restricted-names        | RESTRICTED_NAMES        |                          | names prohibited to use by the user, _multi_    |
 | edit-time               | EDIT_TIME               | `5m`                     | edit window                                     |
 | read-age                | READONLY_AGE            |                          | read-only age of comments, days                 |
 | image-proxy.http2https  |  IMAGE_PROXY_HTTP2HTTPS | `false`                  | enable http->https proxy for images             |
 | image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false`            | enable caching external images to current image storage |
 | emoji                   | EMOJI                   | `false`                  | enable emoji support                            |
 | simple-view             | SIMPLE_VIEW             | `false`                  | minimized UI with basic info only               |
+| proxy-cors              | PROXY_CORS              | `false`                  | disable internal CORS and delegate it to proxy  |
+| allowed-hosts           | ALLOWED_HOSTS           |  enable all              | limit hosts/sources allowed to embed comments   |
 | port                    | REMARK_PORT             | `8080`                   | web server port                                 |
 | web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                       |
 | update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                               |
@@ -283,6 +294,14 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
 1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
 
+#### Microsoft Auth Provider
+
+1.  Register a new application [using the Azure portal](https://docs.microsoft.com/en-us/graph/auth-register-app-v2).
+2.  Under **"Authentication/Platform configurations/Web"** enter the correct url constructed as domain + `/auth/microsoft/callback`. i.e. `https://example.mysite.com/auth/microsoft/callback`
+3.  In "Overview" take note of the **Application (client) ID**
+4.  Choose the new project from the top right project dropdown (only if another project is selected)
+5.  Select "Certificates & secrets" and click on "+ New Client Secret".
+
 ##### Twitter Auth Provider
 
 1.	Create a new twitter application https://developer.twitter.com/en/apps
@@ -311,11 +330,16 @@ Optionally, anonymous access can be turned on. In this case an extra `anonymous`
 - name should be at least 3 characters long
 - name has to start from the letter and contains letters, numbers, underscores and spaces only.
 
+### Importing comments
+
+Remark supports importing comments from Disqus, WordPress or native backup format.
+All imported comments has `Imported` field set to `true`.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
 2.  Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
+3.  Run import command - `docker exec -it remark42 import -p disqus -f /srv/var/{disqus-export-name}.xml -s {your site id}`
 
 #### Initial import from WordPress
 
@@ -363,14 +387,14 @@ It will expand login info and show full user ID.
 
 #### Docker parameters
 
-Two parameters allow to customize docker container on the system level:
+Two parameters allow customizing Docker container on the system level:
 
 - `APP_UID` - sets UID to run remark42 application in container (default=1001)
 - `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
 
 _see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
 
-example of compose:
+example of `docker-compose.yml`:
 
 ```yaml
 version: '2'
@@ -434,18 +458,15 @@ Add this snippet to the bottom of web page:
                      // in well defined order
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
-    page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
+    page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
+    locale: 'en', // set up locale and language, if it isn't defined default value ('en') will be used
+    show_email_subscription: false // optional param; by default it is `true` and you can see email subscription feature
+                                   // in interface when enable it from backend side
+                                   // if you set this param in `false` you will get notifications email notifications as admin
+                                   // but your users won't have interface for subscription
   };
-
-  (function(c) {
-    for(var i = 0; i < c.length; i++){
-      var d = document, s = d.createElement('script');
-      s.src = remark_config.host + '/web/' +c[i] +'.js';
-      s.defer = true;
-      (d.head || d.body).appendChild(s);
-    }
-  })(remark_config.components || ['embed']);
 </script>
+<script>!function(e,n){for(var o=0;o<e.length;o++){var r=n.createElement("script"),c=".js",d=n.head||n.body;"noModule"in r?(r.type="module",c=".mjs"):r.async=!0,r.defer=!0,r.src=remark_config.host+"/web/"+e[o]+c,d.appendChild(r)}}(remark_config.components||["embed"],document);</script>
 ```
 
 And then add this node in the place where you want to see Remark42 widget:
@@ -456,6 +477,8 @@ And then add this node in the place where you want to see Remark42 widget:
 
 After that widget will be rendered inside this node.
 
+If you want to set this up on a Single Page App, see [appropriate doc page](https://remark42.com/docs/latest/spa/).
+
 ##### Themes
 
 Right now Remark has two themes: light and dark.
@@ -469,6 +492,13 @@ Just call this function and pass a name of the theme that you want to turn on:
 window.REMARK42.changeTheme('light');
 ```
 
+##### Locales
+
+Right now Remark is translated to en, ru (partially), de, and fi languages.
+You can pick one using [configuration object](#setup-on-your-website).
+
+Do you want translate remark42 to other locale? Please see [this documentation](https://github.com/umputun/remark42/blob/master/docs/translation.md) for details.
+
 #### Last comments
 
 It's a widget which renders list of last comments from your site.
@@ -482,15 +512,6 @@ Add this snippet to the bottom of web page, or adjust already present `remark_co
     site_id: 'YOUR_SITE_ID',
     components: ['last-comments']
   };
-
-  (function(c) {
-    for(var i = 0; i < c.length; i++){
-      var d = document, s = d.createElement('script');
-      s.src = remark_config.host + '/web/' +c[i] +'.js';
-      s.defer = true;
-      (d.head || d.body).appendChild(s);
-    }
-  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -515,15 +536,6 @@ Add this snippet to the bottom of web page, or adjust already present `remark_co
     site_id: 'YOUR_SITE_ID',
     components: ['counter']
   };
-
-  (function(c) {
-    for(var i = 0; i < c.length; i++){
-      var d = document, s = d.createElement('script');
-      s.src = remark_config.host + '/web/' +c[i] +'.js';
-      s.defer = true;
-      (d.head || d.body).appendChild(s);
-    }
-  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -541,23 +553,26 @@ Also script can use `url` property from `remark_config` object, or `window.locat
 
 ## Build from the source
 
-- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
+- to build Docker container - `make docker`. This command will produce container `umputun/remark42`.
 - to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
  `remark42` file with everything embedded.
 
 ## Development
 
-You can use fully functional local version to develop and test both frontend & backend.
+You can use fully functional local version to develop and test both frontend & backend. It requires at least 2GB RAM or swap enabled
 
 To bring it up run:
 
 ```bash
 # if you mainly work on backend
-docker-compose -f compose-dev-backend.yml build
-docker-compose -f compose-dev-backend.yml up
+cp compose-dev-backend.yml compose-private.yml
 # if you mainly work on frontend
-docker-compose -f compose-dev-frontend.yml build
-docker-compose -f compose-dev-frontend.yml up
+cp compose-dev-frontend.yml compose-private.yml
+# now, edit / debug `compose-private.yml` to your heart's content.
+
+# build and run
+docker-compose -f compose-private.yml build
+docker-compose -f compose-private.yml up
 ```
 
 It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
@@ -565,29 +580,34 @@ To access UI demo page go to `127.0.0.1:8080/web`.
 By default, you would be logged in as `dev_user` which defined as admin.
 You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
 
-Backend docker compose config by default skips running frontend related tests.
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
+Backend Docker Compose config by default skips running frontend related tests.
+Frontend Docker Compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
 
 ### Backend development
 
-In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
+In order to run backend locally (development mode, without Docker) you have to have the latest stable `go` toolchain [installed](https://golang.org/doc/install).
 
-To run backend - `go run backend/app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
+To run backend - `cd backend; go run app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
 It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
 `HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
 
 ### Frontend development
 
-#### Build
+#### Developer guide
 
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
+#### Build
+You should have at least 2GB RAM or swap enabled for building
+
+* install [Node.js 12.11](https://nodejs.org/en/) or higher;
+* install [NPM 6.13.4](https://www.npmjs.com/package/npm);
 * run `npm install` inside `./frontend`;
 * run `npm run build` there;
 * result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
-It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
+It used to reformat your frontend code using `prettier` and lint with `eslint` and `stylelint` before every commit.
 
 #### Devserver
 
@@ -602,12 +622,20 @@ You can attach to locally running backend by providing `REMARK_URL` environment
 npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 ```
 
+**Note** If you want to redefine env variables such as `PORT` on your local instance you can add `.env` file
+to `./frontend` folder and rewrite variables as you wish. For such functional we use `dotenv`
+
+The best way for start local developer environment:
+```sh
+cp compose-dev-frontend.yml compose-private-frontend.yml
+docker-compose -f compose-private-frontend.yml up --build
+cd frontend
+npm run dev
+```
+
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
-#### Frontend guide
-
-Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
 
 ## API
 
@@ -860,5 +888,6 @@ _all admin calls require auth and admin privilege_
 * User can edit comments in 5 mins (configurable) window after creation.
 * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
 * All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
-* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
+* Images can be proxied (`IMAGE_PROXY_HTTP2HTTPS=true`) to prevent mixed http/https.
+* All images can be proxied and saved (`IMAGE_PROXY_CACHE_EXTERNAL=true`) instead of serving from original location. Beware, images which are posted with this parameter enabled will be served from proxy even after it will be disabled.
 * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.

SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| current | :white_check_mark: 
+| 1.6.x   | :white_check_mark: |
+| <1.5.x | :x:                 |
+
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to umputun@gmail.com. You will receive a response from us within 48 hours.
+If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.
+

backend/.golangci.yml

@@ -1,11 +1,42 @@
 run:
-  tests: false
+  timeout: 5m
   output:
     format: tab
   skip-dirs:
     - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0.1
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+# TODO: feel free to remove these excludes and fix the code
+      - hugeParam
+      - rangeValCopy
+      - singleCaseSwitch
+      - ifElseChain
+
 linters:
   enable:
+    - megacheck
+    - golint
+    - govet
     - unconvert
     - megacheck
     - structcheck
@@ -19,4 +50,29 @@ linters:
     - typecheck
     - ineffassign
     - varcheck
+    - stylecheck
+    - gochecknoinits
+    - scopelint
+    - gocritic
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
   disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "should have a package comment, unless it's in another file for this package"
+      linters:
+        - golint
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false
+
+service:
+  golangci-lint-version: 1.31.x

backend/_example/memory_store/README.md

@@ -4,10 +4,10 @@
 
 In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
 
-1. docker-compose -f compose-dev-memstore.yml build
-1. docker-compose -f compose-dev-memstore.yml up
+1. `docker-compose -f compose-dev-memstore.yml build`
+1. `docker-compose -f compose-dev-memstore.yml up`
 
 As usual, demo site will run on http://127.0.0.1:8080/web/
 
-note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package
-. In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
+In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -9,7 +9,8 @@ package accessor
 import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
-	"github.com/umputun/remark/backend/app/store/admin"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
 )
 
 // MemAdmin implements admin.Store with memory backend
@@ -34,7 +35,7 @@ func NewMemAdminStore(key string) *MemAdmin {
 }
 
 // Key executes find by siteID and returns substructure with secret key
-func (m *MemAdmin) Key() (key string, err error) {
+func (m *MemAdmin) Key(_ string) (key string, err error) {
 	return m.key, nil
 }
 
@@ -74,7 +75,7 @@ func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
 		return errors.Errorf("site %s not found", siteID)
 	}
 	if ev == admin.EvCreate {
-		resp.CountCreated += 1 // not a good idea, just for demo
+		resp.CountCreated++ // not a good idea, just for demo
 		m.data[siteID] = resp
 	}
 	return nil

backend/_example/memory_store/accessor/admin_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/admin"
 )
 
 func TestMemAdmin_Get(t *testing.T) {
@@ -31,7 +31,7 @@ func TestMemAdmin_Get(t *testing.T) {
 	email, err := ms.Email("site1")
 	assert.NoError(t, err)
 	assert.Equal(t, "e1", email)
-	key, err := ms.Key()
+	key, err := ms.Key("any")
 	assert.NoError(t, err)
 	assert.Equal(t, "secret", key)
 
@@ -41,15 +41,17 @@ func TestMemAdmin_Get(t *testing.T) {
 	email, err = ms.Email("site2")
 	assert.NoError(t, err)
 	assert.Equal(t, "e2", email)
-	key, err = ms.Key()
+	key, err = ms.Key("any")
 	assert.NoError(t, err)
 	assert.Equal(t, "secret", key)
 
 	admins, err = ms.Admins("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
 
 	email, err = ms.Email("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
 
 	enabled, err := ms.Enabled("site1")
 	assert.NoError(t, err)
@@ -61,6 +63,7 @@ func TestMemAdmin_Get(t *testing.T) {
 
 	enabled, err = ms.Enabled("no-site-in-db")
 	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
 
 	err = ms.OnEvent("site1", admin.EvCreate)
 	assert.NoError(t, err)

backend/_example/memory_store/accessor/data.go

@@ -14,9 +14,8 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store/engine"
-
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
 )
 
 const lastLimit = 1000
@@ -536,18 +535,19 @@ func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error
 func (m *MemData) updateComment(comment store.Comment) error {
 	comments := m.posts[comment.Locator.SiteID]
 	for i, c := range comments {
-		if c.ID == comment.ID && c.Locator == comment.Locator {
-			c.Text = comment.Text
-			c.Orig = comment.Orig
-			c.Score = comment.Score
-			c.Votes = comment.Votes
-			c.Pin = comment.Pin
-			c.Deleted = comment.Deleted
-			c.User = comment.User
-			comments[i] = c
-			m.posts[comment.Locator.SiteID] = comments
-			return nil
+		if c.ID != comment.ID || c.Locator != comment.Locator {
+			continue
 		}
+		c.Text = comment.Text
+		c.Orig = comment.Orig
+		c.Score = comment.Score
+		c.Votes = comment.Votes
+		c.Pin = comment.Pin
+		c.Deleted = comment.Deleted
+		c.User = comment.User
+		comments[i] = c
+		m.posts[comment.Locator.SiteID] = comments
+		return nil
 	}
 	return errors.New("not found")
 }

backend/_example/memory_store/accessor/data_test.go

@@ -15,8 +15,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
 )
 
 func TestMemData_CreateAndFind(t *testing.T) {
@@ -425,6 +425,7 @@ func TestMemData_FlagReadOnlyPost(t *testing.T) {
 		Update: engine.FlagTrue}
 	val, err = b.Flag(req)
 	assert.NoError(t, err)
+	assert.True(t, val)
 	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
 	val, err = b.Flag(req)
 	assert.NoError(t, err)
@@ -546,9 +547,9 @@ func TestMemData_FlagListBlocked(t *testing.T) {
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	var blockedIds []string
-	for _, x := range blockedList {
-		blockedIds = append(blockedIds, x.ID)
+	var blockedIds = make([]string, len(blockedList))
+	for i, x := range blockedList {
+		blockedIds[i] = x.ID
 	}
 	require.Equal(t, 2, len(blockedList), b.metaUsers)
 	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.Unlock()
+
+	return nil
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.RUnlock()
+	if !ok {
+		return nil, errors.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.RLock()
+	img, ok := m.imagesStaging[id]
+	m.RUnlock()
+	if !ok {
+		return errors.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.Lock()
+	m.images[id] = img
+	m.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.RUnlock()
+
+	m.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -7,16 +7,22 @@ services:
 
   remark42:
     build:
-      context: .
+      context: ../../..
       dockerfile: Dockerfile
       args:
         - SKIP_BACKEND_TEST=true
         - SKIP_FRONTEND_TEST=true
     image: umputun/remark42:dev
-    container_name: "remark42"
-    hostname: "remark42"
+    container_name: "remark42-dev"
+    hostname: "remark42-dev"
     restart: always
 
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
     ports:
       - "8080:8080" # primary rest server
       - "8084:8084" # local oauth2 server
@@ -33,14 +39,26 @@ services:
       - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
       - STORE_TYPE=rpc
       - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
 
   mem_store.r42:
     image: umputun/mem_store.r42
     build:
-      context: .
+      context: ../../..
       dockerfile: backend/_example/memory_store/Dockerfile
     container_name: "mem_store.r42"
     hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
     environment:
       - API=/cmd
       - SECRET=123456

backend/_example/memory_store/go.mod

@@ -1,16 +1,14 @@
-module github.com/umputun/remark/memory_store
+module github.com/umputun/remark42/memory_store
 
-go 1.12
+go 1.14
 
 require (
-	github.com/go-pkgz/jrpc v0.1.0
-	github.com/go-pkgz/lgr v0.6.3
-	github.com/jessevdk/go-flags v1.4.0
-	github.com/pkg/errors v0.8.1
-	github.com/stretchr/testify v1.4.0
-	github.com/umputun/remark/backend v1.4.0
+	github.com/go-pkgz/jrpc v0.2.0
+	github.com/go-pkgz/lgr v0.10.4
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.6.1
+	github.com/umputun/go-flags v1.5.1
+	github.com/umputun/remark42/backend v1.6.0
 )
 
-replace github.com/umputun/remark/backend => ../../
-
-replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1
+replace github.com/umputun/remark42/backend => ../../

backend/_example/memory_store/go.sum

@@ -1,330 +1,296 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
-github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
-github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
+github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
+github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
+github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
-github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
-github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/alicebob/miniredis/v2 v2.11.4/go.mod h1:VL3UDEfAH59bSa7MuHMuFToxkqyHh69s/WUbYlOAuyg=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
-github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
-github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
-github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
-github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/didip/tollbooth/v6 v6.0.1 h1:QvLvRpB1G2bzKvkRze0muMUBlGN9H1z7tJ4DH4ypWOU=
+github.com/didip/tollbooth/v6 v6.0.1/go.mod h1:j2pKs+JQ5PvU/K4jFnrnwntrmfUbYLJE5oSdxR37FD0=
+github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9 h1:gTh8fKuI/yLqQtZEPlDX3ZGsiTPZIe0ADHsxXSbwO1I=
+github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
-github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
-github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
-github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
+github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-pkgz/auth v0.9.0/go.mod h1:8Kiold3xA9VUiYZwMyMhwBgiGImZFoL/EmtO9lSIQxo=
-github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
-github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
-github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
-github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
-github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/auth v0.11.0/go.mod h1:NzVqlTW0E9JXVdAaWRq81XZjICgHnNaNdUfE3CbS2T4=
+github.com/go-pkgz/auth v1.14.0/go.mod h1:1GVd61pXZcuJ0ZnOUdCTY08V8SreO7MJtsvEd5/WEWA=
+github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
+github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
+github.com/go-pkgz/jrpc v0.2.0 h1:CLy/eZyekjraVrxZV18N2R1mYLMJ/nWrgdfyIOGPY/E=
+github.com/go-pkgz/jrpc v0.2.0/go.mod h1:wd8vtQ4CgtCnuqua6x2b1SKIgv0VSOh5Dn0uUITbiUE=
+github.com/go-pkgz/lcw v0.7.1/go.mod h1:3P6g9QrJsDePXEMe42ywO+tW08L17tBJGwIDdI7lZ6g=
+github.com/go-pkgz/lcw v0.8.1/go.mod h1:Xw0/ZfApATgbjVPYRZO4XHdWyxAjErDWDWJ7TLlw1Vc=
+github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
+github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
+github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
+github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
 github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
-github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
-github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
 github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
-github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
+github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.7.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
-github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
-github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
 github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
-github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
 github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
-github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a h1:Yu34BogBivvmu7SAzHHaB9nZWH5D1C+z3F1jyIaYZSQ=
-golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
-gopkg.in/oauth2.v3 v3.11.0/go.mod h1:ZdKwOng79bFq17t/ItHGtL8LJ2dW3tmrdJJhHmqS+uc=
+gopkg.in/oauth2.v3 v3.12.0/go.mod h1:XEYgKqWX095YiPT+Aw5y3tCn+7/FMnlTFKrupgSiJ3I=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/main.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Umputun. All rights reserved.
+ * Copyright 2020 Umputun. All rights reserved.
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
@@ -12,10 +12,10 @@ import (
 
 	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
-	"github.com/umputun/remark/memory_store/accessor"
-	"github.com/umputun/remark/memory_store/server"
+	"github.com/umputun/remark42/memory_store/accessor"
+	"github.com/umputun/remark42/memory_store/server"
 )
 
 // opts with all cli commands and flags
@@ -41,6 +41,7 @@ func main() {
 
 	dataStore := accessor.NewMemData()
 	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
 
 	rpcServer := jrpc.Server{
 		API:        opts.API,
@@ -51,12 +52,13 @@ func main() {
 		Logger:     log.Default(),
 	}
 
-	srv := server.NewRPC(dataStore, adminStore, &rpcServer)
+	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
 
 	admRec := accessor.AdminRec{
-		SiteID: "remark",
-		IDs:    []string{"dev_user"},
-		Email:  "admin@example.com",
+		SiteID:  "remark",
+		Enabled: true,
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
 	}
 	adminStore.Set("remark", admRec)
 

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	key, err := s.adm.Key(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,306 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/server/rpc.go

@@ -1,13 +1,17 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
 package server
 
 import (
-	"encoding/json"
-
 	"github.com/go-pkgz/jrpc"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
 )
 
 // RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
@@ -16,11 +20,12 @@ type RPC struct {
 	*jrpc.Server
 	eng engine.Interface
 	adm admin.Store
+	img image.Store
 }
 
 // NewRPC makes RPC instance and register handlers
-func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC {
-	res := &RPC{eng: e, adm: a, Server: r}
+func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, img: i, Server: r}
 	res.addHandlers()
 	return res
 }
@@ -49,186 +54,13 @@ func (s *RPC) addHandlers() {
 		"enabled": s.admEnabledHndl,
 		"event":   s.admEventHndl,
 	})
-}
 
-func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	comment := store.Comment{}
-	if err := json.Unmarshal(params, &comment); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	commentID, err := s.eng.Create(comment)
-	return jrpc.EncodeResponse(id, commentID, err)
-}
-
-// Find comments
-func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FindRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	resp, err := s.eng.Find(req)
-	return jrpc.EncodeResponse(id, resp, err)
-}
-
-// Get comment
-func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.GetRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	comment, err := s.eng.Get(req)
-	return jrpc.EncodeResponse(id, comment, err)
-}
-
-// Update comment
-func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	comment := store.Comment{}
-	if err := json.Unmarshal(params, &comment); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	err := s.eng.Update(comment)
-	return jrpc.EncodeResponse(id, nil, err)
-}
-
-// counts for site and users
-func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FindRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	count, err := s.eng.Count(req)
-	return jrpc.EncodeResponse(id, count, err)
-}
-
-// info get post meta info
-func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.InfoRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	info, err := s.eng.Info(req)
-	return jrpc.EncodeResponse(id, info, err)
-}
-
-// flagHndl get and sets flag value
-func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FlagRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	status, err := s.eng.Flag(req)
-	return jrpc.EncodeResponse(id, status, err)
-}
-
-// listFlagsHndl list flags for given request
-func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.FlagRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	flags, err := s.eng.ListFlags(req)
-	return jrpc.EncodeResponse(id, flags, err)
-}
-
-// userDetailHndl sets or gets single detail value, or gets all details for requested site.
-// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
-// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
-func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.UserDetailRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	value, err := s.eng.UserDetail(req)
-	return jrpc.EncodeResponse(id, value, err)
-}
-
-// deleteHndl delete post(s), user, comment, user details, or everything
-func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	req := engine.DeleteRequest{}
-	if err := json.Unmarshal(params, &req); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	err := s.eng.Delete(req)
-	return jrpc.EncodeResponse(id, nil, err)
-}
-
-// close store
-func (s *RPC) closeHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
-	if err := s.eng.Close(); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.Response{}
-}
-
-// get admin key
-func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
-	key, err := s.adm.Key()
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, key, err)
-}
-
-// get admins list
-func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	admins, err := s.adm.Admins(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, admins, err)
-}
-
-// get admin email
-func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	email, err := s.adm.Email(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, email, err)
-}
-
-// return site enabled status
-func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	if err := json.Unmarshal(params, &siteID); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-
-	ok, err := s.adm.Enabled(siteID)
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, ok, err)
-}
-
-// onEvent returns nothing, callback to OnEvent
-func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
-	var siteID string
-	ps := []interface{}{}
-	if err := json.Unmarshal(params, &ps); err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	siteID, ok := ps[0].(string)
-	if !ok {
-		return jrpc.Response{Error: "wrong siteID type"}
-	}
-	evType, ok := ps[1].(float64)
-	if !ok {
-		return jrpc.Response{Error: "wrong event type"}
-	}
-	err := s.adm.OnEvent(siteID, admin.EventType(evType))
-	if err != nil {
-		return jrpc.Response{Error: err.Error()}
-	}
-	return jrpc.EncodeResponse(id, nil, err)
+	// image store handlers
+	s.Group("image", jrpc.HandlersGroup{
+		"save_with_id": s.imgSaveWithIDHndl,
+		"load":         s.imgLoadHndl,
+		"commit":       s.imgCommitHndl,
+		"cleanup":      s.imgCleanupHndl,
+		"info":         s.imgInfoHndl,
+	})
 }

backend/_example/memory_store/server/rpc_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Umputun. All rights reserved.
+ * Copyright 2020 Umputun. All rights reserved.
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
@@ -15,370 +15,11 @@ import (
 	"time"
 
 	"github.com/go-pkgz/jrpc"
-	log "github.com/go-pkgz/lgr"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
 
-	"github.com/umputun/remark/memory_store/accessor"
+	"github.com/umputun/remark42/memory_store/accessor"
 )
 
-func TestRPC_createHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-}
-
-func TestRPC_findHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	comments, err := re.Find(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 0, len(comments))
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	comments, err = re.Find(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, len(comments))
-	assert.Equal(t, c, comments[0])
-}
-
-func TestRPC_getHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	req := engine.GetRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-
-	_, err := re.Get(req)
-	assert.EqualError(t, err, "not found")
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	comment, err := re.Get(req)
-	assert.NoError(t, err)
-	assert.Equal(t, c, comment)
-}
-
-func TestRPC_updateHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	err := re.Update(c)
-	assert.EqualError(t, err, "not found")
-
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	c.Text = "updates"
-	err = re.Update(c)
-	assert.NoError(t, err)
-
-	req := engine.GetRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-	comment, err := re.Get(req)
-	assert.NoError(t, err)
-	assert.Equal(t, c, comment)
-}
-
-func TestRPC_countHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	count, err := re.Count(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 0, count)
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	count, err = re.Count(findReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, count)
-}
-
-func TestRPC_infoHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
-	info, err := re.Info(infoReq)
-	require.NoError(t, err)
-	assert.Equal(t, 1, len(info))
-	i := info[0]
-	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
-}
-
-func TestRPC_flagHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	flagReq := engine.FlagRequest{
-		Flag: engine.Verified,
-		Locator: store.Locator{
-			SiteID: "test-site",
-		},
-		UserID: "u1",
-	}
-	status, err := re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, false, status)
-
-	flagReq.Update = engine.FlagTrue
-	status, err = re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-
-	flagReq.Update = engine.FlagNonSet
-	status, err = re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-}
-
-func TestRPC_listFlagsHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	id, err := re.Create(c)
-	assert.NoError(t, err)
-	assert.Equal(t, "123456", id)
-
-	flagReq := engine.FlagRequest{
-		Flag:   engine.Verified,
-		UserID: "u1",
-		Locator: store.Locator{
-			SiteID: "test-site",
-		},
-	}
-	flags, err := re.ListFlags(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, []interface{}{}, flags)
-
-	flagReq.Update = engine.FlagTrue
-	status, err := re.Flag(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, true, status)
-
-	flags, err = re.ListFlags(flagReq)
-	require.NoError(t, err)
-	assert.Equal(t, []interface{}{"u1"}, flags)
-}
-
-func TestRPC_userDetailHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-
-	// add to entries to DB before we start
-	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
-	assert.NoError(t, err, "No error inserting entry expected")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
-	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
-	assert.NoError(t, err, "No error inserting entry expected")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
-
-	// try to change existing entry with wrong SiteID
-	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
-	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
-	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
-
-	// stateless tests without changing the state we set up before
-	var testData = []struct {
-		req      engine.UserDetailRequest
-		error    string
-		expected []engine.UserDetailEntry
-	}{
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{}},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
-			expected: []engine.UserDetailEntry{}},
-		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
-			error: `userid cannot be empty in request for single detail`},
-		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
-			error: `unsupported detail "bad"`},
-		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
-			error: `unsupported request with userdetail all`},
-		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
-			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
-	}
-
-	for i, x := range testData {
-		result, err := re.UserDetail(x.req)
-		if x.error != "" {
-			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
-		} else {
-			assert.NoError(t, err, "Error is not expected expected for case %d", i)
-		}
-		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
-	}
-}
-
-func TestRPC_deleteHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	req := engine.DeleteRequest{
-		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		CommentID: "123456",
-	}
-
-	err := re.Delete(req)
-	assert.EqualError(t, err, "not found")
-
-	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
-		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
-	_, err = re.Create(c)
-	assert.NoError(t, err)
-
-	err = re.Delete(req)
-	assert.NoError(t, err)
-}
-
-func TestRPC_closeHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := re.Close()
-	assert.NoError(t, err)
-}
-func TestRPC_admKeyHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	key, err := ra.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-}
-
-func TestRPC_admAdminsHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Admins("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	admins, err := ra.Admins("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"id1", "id2"}, admins)
-}
-
-func TestRPC_admEmailHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Admins("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	email, err := ra.Email("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, "admin@example.com", email)
-}
-
-func TestRPC_admEnabledHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	_, err := ra.Enabled("bad site")
-	assert.EqualError(t, err, "site bad site not found")
-
-	ok, err := ra.Enabled("test-site")
-	assert.NoError(t, err)
-	assert.Equal(t, true, ok)
-
-	ok, err = ra.Enabled("test-site-disabled")
-	assert.NoError(t, err)
-	assert.Equal(t, false, ok)
-}
-
-func TestRPC_admEventHndl(t *testing.T) {
-	_, port, teardown := prepTestStore(t)
-	defer teardown()
-	api := fmt.Sprintf("http://localhost:%d/test", port)
-
-	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := ra.OnEvent("bad site", admin.EvCreate)
-	assert.EqualError(t, err, "site bad site not found")
-
-	err = ra.OnEvent("test-site", admin.EvCreate)
-	assert.NoError(t, err)
-}
-
 func chooseRandomUnusedPort() (port int) {
 	for i := 0; i < 10; i++ {
 		port = 40000 + int(rand.Int31n(10000))
@@ -402,10 +43,11 @@ func waitForHTTPServerStart(port int) {
 	}
 }
 
-func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
+func prepTestStore(t *testing.T) (port int, teardown func()) {
 	mg := accessor.NewMemData()
 	adm := accessor.NewMemAdminStore("secret")
-	s = NewRPC(mg, adm, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+	img := accessor.NewMemImageStore()
+	s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
 
 	admRec := accessor.AdminRec{
 		SiteID:  "test-site",
@@ -421,12 +63,12 @@ func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
 
 	port = chooseRandomUnusedPort()
 	go func() {
-		log.Printf("%v", s.Run(port))
+		_ = s.Run(port)
 	}()
 
 	waitForHTTPServerStart(port)
 
-	return s, port, func() {
+	return port, func() {
 		require.NoError(t, s.Shutdown())
 	}
 }

backend/app/cmd/avatar.go

@@ -3,9 +3,9 @@ package cmd
 import (
 	"path"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth/avatar"
 )
@@ -34,7 +34,7 @@ func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 }
 
 // Execute runs  with AvatarCommand parameters, entry point for "avatar" command
-func (ac *AvatarCommand) Execute(args []string) error {
+func (ac *AvatarCommand) Execute(_ []string) error {
 	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
 
 	src, err := ac.makeAvatarStore(ac.AvatarSrc)
@@ -72,12 +72,12 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	switch gr.Type {
 	case "fs":
 		if err := makeDirs(gr.FS.Path); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "failed to create avatar store")
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "failed to create avatar store")
 		}
 		return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
 	}

backend/app/cmd/avatar_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 
 	"github.com/go-pkgz/auth/avatar"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 )
 
 func TestAvatar_Execute(t *testing.T) {
@@ -42,7 +42,7 @@ type avatarMigratorMock struct {
 	retCount int
 }
 
-func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
+func (a *avatarMigratorMock) Migrate(_, _ avatar.Store) (int, error) {
 	a.called++
 	return a.retCount, a.retError
 }

backend/app/cmd/backup.go

@@ -24,7 +24,7 @@ type BackupCommand struct {
 }
 
 // Execute runs export with ExportCommand parameters, entry point for "export" command
-func (ec *BackupCommand) Execute(args []string) error {
+func (ec *BackupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 

backend/app/cmd/backup_test.go

@@ -8,7 +8,8 @@ import (
 	"os"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/cleanup.go

@@ -11,7 +11,7 @@ import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // CleanupCommand set of flags and command for cleanup
@@ -34,7 +34,7 @@ var (
 
 // Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
 // This command uses provided flags to detect and remove junk comments
-func (cc *CleanupCommand) Execute(args []string) error {
+func (cc *CleanupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] cleanup for site %s", cc.Site)
 
 	posts, err := cc.postsInRange(cc.From, cc.To)
@@ -234,7 +234,7 @@ func (cc *CleanupCommand) setTitle(c store.Comment) error {
 }
 
 // isSpam calculates spam's probability as a score
-func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
+func (cc *CleanupCommand) isSpam(comment store.Comment) (isSpam bool, spamScore float64) {
 
 	badWord := func(txt string) float64 {
 		res := 0.0

backend/app/cmd/cleanup_test.go

@@ -10,11 +10,11 @@ import (
 	"time"
 
 	"github.com/go-chi/chi"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 type cleanedComments struct {
@@ -46,6 +46,7 @@ func TestCleanup_IsSpam(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
 		t.Run(checkName, func(t *testing.T) {
 			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
@@ -144,7 +145,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 }
 
 func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
-	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
 		list := []store.PostInfo{
@@ -165,9 +166,9 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 			},
 		}
 		require.NoError(t, json.NewEncoder(w).Encode(list))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/find", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "remark", r.URL.Query().Get("site"))
 		require.Equal(t, "plain", r.URL.Query().Get("format"))
@@ -193,22 +194,22 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		}
 
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
 }

backend/app/cmd/import.go

@@ -26,7 +26,7 @@ type ImportCommand struct {
 }
 
 // Execute runs import with ImportCommand parameters, entry point for "import" command
-func (ic *ImportCommand) Execute(args []string) error {
+func (ic *ImportCommand) Execute(_ []string) error {
 	log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 
@@ -69,7 +69,7 @@ func (ic *ImportCommand) Execute(args []string) error {
 
 // reader returns reader for file. For .gz file wraps with gunzip
 func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
-	inpFile, err := os.Open(inp)
+	inpFile, err := os.Open(inp) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
 	}

backend/app/cmd/import_test.go

@@ -9,7 +9,8 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/remap.go

@@ -23,7 +23,7 @@ type RemapCommand struct {
 }
 
 // Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
-func (rc *RemapCommand) Execute(args []string) error {
+func (rc *RemapCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 

backend/app/cmd/remap_test.go

@@ -6,7 +6,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/restore_test.go

@@ -7,7 +7,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

backend/app/cmd/server.go

@@ -14,13 +14,13 @@ import (
 	"syscall"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/jrpc"
+	"github.com/go-pkgz/lcw/eventbus"
 	log "github.com/go-pkgz/lgr"
 	"github.com/kyokomi/emoji"
-	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
@@ -29,15 +29,16 @@ import (
 	"github.com/go-pkgz/auth/token"
 	cache "github.com/go-pkgz/lcw"
 
-	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/notify"
-	"github.com/umputun/remark/backend/app/rest/api"
-	"github.com/umputun/remark/backend/app/rest/proxy"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/image"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/migrator"
+	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest/api"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/templates"
 )
 
 // ServerCommand with command line flags and env
@@ -47,10 +48,9 @@ type ServerCommand struct {
 	Cache      CacheGroup      `group:"cache" namespace:"cache" env-namespace:"CACHE"`
 	Admin      AdminGroup      `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify     NotifyGroup     `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
-	SMTP       SmtpGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
+	SMTP       SMTPGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
 	Image      ImageGroup      `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
-	Stream     StreamGroup     `group:"stream" namespace:"stream" env-namespace:"STREAM"`
 	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
 	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -72,17 +72,25 @@ type ServerCommand struct {
 	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
 	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedNames  []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
 	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
 	SimpleView       bool          `long:"simpler-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
+	ProxyCORS        bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
+	AllowedHosts     []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments"`
 
 	Auth struct {
 		TTL struct {
 			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
 			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
 		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
+
+		SendJWTHeader bool   `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"send JWT as a header instead of cookie"`
+		SameSite      string `long:"same-site" env:"SAME_SITE" description:"set same site policy for cookies" choice:"default" choice:"none" choice:"lax" choice:"strict" default:"default"` // nolint
+
 		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
 		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
 		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Microsoft AuthGroup `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
 		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
 		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
 		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
@@ -98,7 +106,7 @@ type ServerCommand struct {
 			SMTPUserName string        `long:"user" env:"USER" description:"[deprecated, use --smtp.username] enable TLS"`
 			TLS          bool          `long:"tls" env:"TLS" description:"[deprecated, use --smtp.tls] SMTP TCP connection timeout"`
 			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"[deprecated, use --smtp.timeout] SMTP TCP connection timeout"`
-			MsgTemplate  string        `long:"template" env:"TEMPLATE" description:"message template file"`
+			MsgTemplate  string        `long:"template" env:"TEMPLATE" description:"[deprecated, message template file]" default:"email_confirmation_login.html.tmpl"`
 		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
@@ -129,7 +137,7 @@ type StoreGroup struct {
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"rpc" default:"fs"` // nolint
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -138,9 +146,10 @@ type ImageGroup struct {
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
-	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
-	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
+	MaxSize      int      `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int      `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
+	ResizeHeight int      `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
+	RPC          RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // AvatarGroup defines options group for avatar params
@@ -158,8 +167,9 @@ type AvatarGroup struct {
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` // nolint
-	Max  struct {
+	Type      string `long:"type" env:"TYPE" description:"type of cache" choice:"redis_pub_sub" choice:"mem" choice:"none" default:"mem"` // nolint
+	RedisAddr string `long:"redis_addr" env:"REDIS_ADDR" default:"127.0.0.1:6379" description:"address of redis cache, turn redis cache on for distributed cache"`
+	Max       struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
 		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
@@ -171,13 +181,13 @@ type AdminGroup struct {
 	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"` //nolint
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
-		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
+		Email  []string `long:"email" env:"EMAIL" description:"admin emails" env-delim:","`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
 	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
-// SmtpGroup defines options for SMTP server connection, used in auth and notify modules
-type SmtpGroup struct {
+// SMTPGroup defines options for SMTP server connection, used in auth and notify modules
+type SMTPGroup struct {
 	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
 	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
 	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
@@ -197,8 +207,9 @@ type NotifyGroup struct {
 		API     string        `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
 	} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
 	Email struct {
-		From                string `long:"fromAddress" env:"FROM" description:"from email address"`
+		From                string `long:"from_address" env:"FROM" description:"from email address"`
 		VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
+		AdminNotifications  bool   `long:"notify_admin" env:"ADMIN" description:"notify admin on new comments via ADMIN_SHARED_EMAIL"`
 	} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 }
 
@@ -212,13 +223,6 @@ type SSLGroup struct {
 	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
 }
 
-// StreamGroup define options for streaming apis
-type StreamGroup struct {
-	RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
-	TimeOut         time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
-	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
-}
-
 // RPCGroup defines options for remote modules (plugins)
 type RPCGroup struct {
 	API          string        `long:"api" env:"API" description:"rpc extension api url"`
@@ -231,6 +235,7 @@ type RPCGroup struct {
 type LoadingCache interface {
 	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
 	Flush(req cache.FlusherRequest)                                        // evict matched records
+	Close() error
 }
 
 // serverApp holds all active objects
@@ -244,11 +249,14 @@ type serverApp struct {
 	avatarStore   avatar.Store
 	notifyService *notify.Service
 	imageService  *image.Service
+	authenticator *auth.Service
 	terminated    chan struct{}
+
+	authRefreshCache *authRefreshCache // stored only to close it properly on shutdown
 }
 
 // Execute is the entry point for "server" command, called by flag parser
-func (s *ServerCommand) Execute(args []string) error {
+func (s *ServerCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start server on port %d", s.Port)
 	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC", "ADMIN_PASSWD")
 
@@ -301,6 +309,9 @@ func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
 		s.SMTP.TimeOut = s.Auth.Email.TimeOut
 		result = append(result, DeprecatedFlag{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"})
 	}
+	if s.Auth.Email.MsgTemplate != "email_confirmation_login.html.tmpl" {
+		result = append(result, DeprecatedFlag{Old: "auth.email.template", RemoveVersion: "1.9.0"})
+	}
 	if s.LegacyImageProxy && !s.ImageProxy.HTTP2HTTPS {
 		s.ImageProxy.HTTP2HTTPS = s.LegacyImageProxy
 		result = append(result, DeprecatedFlag{Old: "img-proxy", New: "image-proxy.http2https", RemoveVersion: "1.7.0"})
@@ -313,7 +324,7 @@ func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
 func (s *ServerCommand) newServerApp() (*serverApp, error) {
 
 	if err := makeDirs(s.BackupLocation); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to create backup store")
 	}
 
 	if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
@@ -335,7 +346,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make pictures store")
 	}
-	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
+	log.Printf("[DEBUG] image service for url=%s, EditDuration=%v", imageService.ImageAPI, imageService.EditDuration)
 
 	dataService := &service.DataStore{
 		Engine:                 storeEngine,
@@ -353,14 +364,21 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 
 	loadingCache, err := s.makeCache()
 	if err != nil {
+		_ = dataService.Close()
 		return nil, errors.Wrap(err, "failed to make cache")
 	}
 
 	avatarStore, err := s.makeAvatarStore()
 	if err != nil {
+		_ = dataService.Close()
 		return nil, errors.Wrap(err, "failed to make avatar store")
 	}
-	authenticator := s.makeAuthenticator(dataService, avatarStore, adminStore)
+	authRefreshCache := newAuthRefreshCache()
+	authenticator, err := s.makeAuthenticator(dataService, avatarStore, adminStore, authRefreshCache)
+	if err != nil {
+		_ = dataService.Close()
+		return nil, errors.Wrap(err, "failed to make authenticator")
+	}
 
 	exporter := &migrator.Native{DataStore: dataService}
 
@@ -370,7 +388,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
-		UrlMapperMaker:    migrator.NewUrlMapper,
+		URLMapperMaker:    migrator.NewURLMapper,
 		KeyStore:          adminStore,
 	}
 
@@ -405,34 +423,33 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
+		_ = dataService.Close()
 		return nil, errors.Wrap(err, "failed to make config of ssl server params")
 	}
 
 	srv := &api.Rest{
-		Version:          s.Revision,
-		DataService:      dataService,
-		WebRoot:          s.WebRoot,
-		RemarkURL:        s.RemarkURL,
-		ImageProxy:       imgProxy,
-		CommentFormatter: commentFormatter,
-		Migrator:         migr,
-		ReadOnlyAge:      s.ReadOnlyAge,
-		SharedSecret:     s.SharedSecret,
-		Authenticator:    authenticator,
-		Cache:            loadingCache,
-		NotifyService:    notifyService,
-		SSLConfig:        sslConfig,
-		UpdateLimiter:    s.UpdateLimit,
-		ImageService:     imageService,
-		Streamer: &api.Streamer{
-			TimeOut:   s.Stream.TimeOut,
-			Refresh:   s.Stream.RefreshInterval,
-			MaxActive: int32(s.Stream.MaxActive),
-		},
+		Version:            s.Revision,
+		DataService:        dataService,
+		WebRoot:            s.WebRoot,
+		RemarkURL:          s.RemarkURL,
+		ImageProxy:         imgProxy,
+		CommentFormatter:   commentFormatter,
+		Migrator:           migr,
+		ReadOnlyAge:        s.ReadOnlyAge,
+		SharedSecret:       s.SharedSecret,
+		Authenticator:      authenticator,
+		Cache:              loadingCache,
+		NotifyService:      notifyService,
+		SSLConfig:          sslConfig,
+		UpdateLimiter:      s.UpdateLimit,
+		ImageService:       imageService,
 		EmailNotifications: emailNotifications,
 		EmojiEnabled:       s.EnableEmoji,
 		AnonVote:           s.AnonymousVote && s.RestrictVoteIP,
 		SimpleView:         s.SimpleView,
+		ProxyCORS:          s.ProxyCORS,
+		AllowedAncestors:   s.AllowedHosts,
+		SendJWTHeader:      s.Auth.SendJWTHeader,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -441,22 +458,25 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if s.Auth.Dev {
 		da, errDevAuth := authenticator.DevAuth()
 		if errDevAuth != nil {
+			_ = dataService.Close()
 			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
 
 	return &serverApp{
-		ServerCommand: s,
-		restSrv:       srv,
-		migratorSrv:   migr,
-		exporter:      exporter,
-		devAuth:       devAuth,
-		dataService:   dataService,
-		avatarStore:   avatarStore,
-		notifyService: notifyService,
-		imageService:  imageService,
-		terminated:    make(chan struct{}),
+		ServerCommand:    s,
+		restSrv:          srv,
+		migratorSrv:      migr,
+		exporter:         exporter,
+		devAuth:          devAuth,
+		dataService:      dataService,
+		avatarStore:      avatarStore,
+		notifyService:    notifyService,
+		imageService:     imageService,
+		authenticator:    authenticator,
+		terminated:       make(chan struct{}),
+		authRefreshCache: authRefreshCache,
 	}, nil
 }
 
@@ -471,28 +491,44 @@ func (a *serverApp) run(ctx context.Context) error {
 		<-ctx.Done()
 		log.Print("[INFO] shutdown initiated")
 		a.restSrv.Shutdown()
-		if a.devAuth != nil {
-			a.devAuth.Shutdown()
-		}
-		if e := a.dataService.Close(); e != nil {
-			log.Printf("[WARN] failed to close data store, %s", e)
-		}
-		if e := a.avatarStore.Close(); e != nil {
-			log.Printf("[WARN] failed to close avatar store, %s", e)
-		}
-		a.notifyService.Close()
-		a.imageService.Close()
-		log.Print("[INFO] shutdown completed")
 	}()
 
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
-		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
+		go a.devAuth.Run(ctx) // dev oauth2 server on :8084
+	}
+
+	// staging images resubmit after restart of the app
+	if e := a.dataService.ResubmitStagingImages(a.Sites); e != nil {
+		log.Printf("[WARN] failed to resubmit comments with staging images, %s", e)
 	}
 
 	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
 
 	a.restSrv.Run(a.Port)
+
+	// shutdown procedures after HTTP server is stopped
+	if a.devAuth != nil {
+		a.devAuth.Shutdown()
+	}
+	if e := a.dataService.Close(); e != nil {
+		log.Printf("[WARN] failed to close data store, %s", e)
+	}
+	if e := a.avatarStore.Close(); e != nil {
+		log.Printf("[WARN] failed to close avatar store, %s", e)
+	}
+	if e := a.restSrv.Cache.Close(); e != nil {
+		log.Printf("[WARN] failed to close rest server cache, %s", e)
+	}
+	if e := a.authRefreshCache.Close(); e != nil {
+		log.Printf("[WARN] failed to close auth authRefreshCache, %s", e)
+	}
+	a.notifyService.Close()
+	// call potentially infinite loop with cancellation after a minute as a safeguard
+	minuteCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	a.imageService.Close(minuteCtx)
+
 	close(a.terminated)
 	return nil
 }
@@ -550,12 +586,12 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	switch s.Avatar.Type {
 	case "fs":
 		if err := makeDirs(s.Avatar.FS.Path); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "failed to create avatar store")
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "failed to create avatar store")
 		}
 		return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
 	case "uri":
@@ -565,39 +601,38 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 }
 
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	imageServiceParams := image.ServiceParams{
+		ImageAPI:     s.RemarkURL + "/api/v1/picture/",
+		ProxyAPI:     s.RemarkURL + "/api/v1/img",
+		EditDuration: s.EditDuration,
+		MaxSize:      s.Image.MaxSize,
+		MaxHeight:    s.Image.ResizeHeight,
+		MaxWidth:     s.Image.ResizeWidth,
+	}
 	switch s.Image.Type {
 	case "bolt":
-		boltImageStore, err := image.NewBoltStorage(
-			s.Image.Bolt.File,
-			s.Image.MaxSize,
-			s.Image.ResizeHeight,
-			s.Image.ResizeWidth,
-			bolt.Options{},
-		)
+		boltImageStore, err := image.NewBoltStorage(s.Image.Bolt.File, bolt.Options{})
 		if err != nil {
 			return nil, err
 		}
-		return &image.Service{
-			Store:    boltImageStore,
-			ImageAPI: s.RemarkURL + "/api/v1/picture/",
-			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
-		}, nil
+		return image.NewService(boltImageStore, imageServiceParams), nil
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "failed to create pictures store")
 		}
-		return &image.Service{
-			Store: &image.FileSystem{
-				Location:   s.Image.FS.Path,
-				Staging:    s.Image.FS.Staging,
-				Partitions: s.Image.FS.Partitions,
-				MaxSize:    s.Image.MaxSize,
-				MaxHeight:  s.Image.ResizeHeight,
-				MaxWidth:   s.Image.ResizeWidth,
-			},
-			ImageAPI: s.RemarkURL + "/api/v1/picture/",
-			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
-		}, nil
+		return image.NewService(&image.FileSystem{
+			Location:   s.Image.FS.Path,
+			Staging:    s.Image.FS.Staging,
+			Partitions: s.Image.FS.Partitions,
+		}, imageServiceParams), nil
+	case "rpc":
+		return image.NewService(&image.RPC{
+			Client: jrpc.Client{
+				API:        s.Image.RPC.API,
+				Client:     http.Client{Timeout: s.Image.RPC.TimeOut},
+				AuthUser:   s.Image.RPC.AuthUser,
+				AuthPasswd: s.Image.RPC.AuthPassword,
+			}}, imageServiceParams), nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
 }
@@ -607,12 +642,15 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 
 	switch s.Admin.Type {
 	case "shared":
-		if s.Admin.Shared.Email == "" { // no admin email, use admin@domain
+		sharedAdminEmail := ""
+		if len(s.Admin.Shared.Email) == 0 { // no admin email, use admin@domain
 			if u, err := url.Parse(s.RemarkURL); err == nil {
-				s.Admin.Shared.Email = "admin@" + u.Host
+				sharedAdminEmail = "admin@" + u.Host
 			}
+		} else {
+			sharedAdminEmail = s.Admin.Shared.Email[0]
 		}
-		return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
+		return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, sharedAdminEmail), nil
 	case "rpc":
 		r := &admin.RPC{Client: jrpc.Client{
 			API:        s.Admin.RPC.API,
@@ -629,6 +667,17 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
 	switch s.Cache.Type {
+	case "redis_pub_sub":
+		redisPubSub, err := eventbus.NewRedisPubSub(s.Cache.RedisAddr, "remark42-cache")
+		if err != nil {
+			return nil, errors.Wrap(err, "cache backend initialization, redis PubSub initialisation")
+		}
+		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+			cache.MaxKeys(s.Cache.Max.Items), cache.EventBus(redisPubSub))
+		if err != nil {
+			return nil, errors.Wrap(err, "cache backend initialization")
+		}
+		return cache.NewScache(backend), nil
 	case "mem":
 		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
@@ -642,29 +691,7 @@ func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-var msgTemplate = `
-<!DOCTYPE html>
-<html>
-<head>
-	<meta name="viewport" content="width=device-width" />
-	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-</head>
-<body>
-<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
-	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
-	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
-	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
-		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
-		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and paste this text into “token” field on comments page</i></p>
-		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
-	</div>
-	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
-</div>
-</body>
-</html>
-`
-
-func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
+func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 
 	providers := 0
 	if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
@@ -679,6 +706,10 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		authenticator.AddProvider("facebook", s.Auth.Facebook.CID, s.Auth.Facebook.CSEC)
 		providers++
 	}
+	if s.Auth.Microsoft.CID != "" && s.Auth.Microsoft.CSEC != "" {
+		authenticator.AddProvider("microsoft", s.Auth.Microsoft.CID, s.Auth.Microsoft.CSEC)
+		providers++
+	}
 	if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" {
 		authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
 		providers++
@@ -707,18 +738,33 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 			ContentType:  s.Auth.Email.ContentType,
 		}
 		sndr := sender.NewEmailClient(params, log.Default())
-		authenticator.AddVerifProvider("email", s.loadEmailTemplate(), sndr)
+		tmpl, err := s.loadEmailTemplate()
+		if err != nil {
+			return err
+		}
+		authenticator.AddVerifProvider("email", tmpl, sndr)
 	}
 
 	if s.Auth.Anonymous {
 		log.Print("[INFO] anonymous access enabled")
-		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
+		var isValidAnonName = regexp.MustCompile(`^[\p{L}\d_ ]+$`).MatchString
 		authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
+
+			// don't allow anon with space prefix or suffix
+			if strings.HasPrefix(user, " ") || strings.HasSuffix(user, " ") {
+				log.Printf("[WARN] name %q has space as a suffix or prefix", user)
+				return false, nil
+			}
+
 			user = strings.TrimSpace(user)
 			if len(user) < 3 {
 				log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
 				return false, nil
 			}
+			if len(user) > 64 {
+				log.Printf("[WARN] name %q is too long, should be up to 64 characters", user)
+				return false, nil
+			}
 
 			if !isValidAnonName(user) {
 				log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
@@ -731,22 +777,29 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 	if providers == 0 {
 		log.Printf("[WARN] no auth providers defined")
 	}
+
+	return nil
 }
 
-// loadEmailTemplate trying to get template from opts MsgTemplate and default to embedded
-// if not defined or failed to load
-func (s *ServerCommand) loadEmailTemplate() string {
-	tmpl := msgTemplate
-	if s.Auth.Email.MsgTemplate != "" {
-		log.Printf("[DEBUG] load email template from %s", s.Auth.Email.MsgTemplate)
-		b, err := ioutil.ReadFile(s.Auth.Email.MsgTemplate)
-		if err == nil {
-			tmpl = string(b)
-		} else {
-			log.Printf("[WARN] failed to load email template from %s, %v", s.Auth.Email.MsgTemplate, err)
-		}
+// loadEmailTemplate trying to get template from statik
+func (s *ServerCommand) loadEmailTemplate() (string, error) {
+	var file []byte
+	var err error
+
+	if s.Auth.Email.MsgTemplate == "email_confirmation_login.html.tmpl" {
+		fs := templates.NewFS()
+		file, err = fs.ReadFile(s.Auth.Email.MsgTemplate)
+	} else {
+		// deprecated loading from an external file, should be removed before v1.9.0
+		file, err = ioutil.ReadFile(s.Auth.Email.MsgTemplate)
+		log.Printf("[INFO] template %s will be read from disk", s.Auth.Email.MsgTemplate)
 	}
-	return tmpl
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to read file %s", s.Auth.Email.MsgTemplate)
+	}
+
+	return string(file), nil
 }
 
 func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *auth.Service) (*notify.Service, error) {
@@ -767,7 +820,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
 				VerificationSubject: s.Notify.Email.VerificationSubject,
 				UnsubscribeURL:      s.RemarkURL + "/email/unsubscribe.html",
 				// TODO: uncomment after #560 frontend part is ready and URL is known
-				//SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
+				// SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
 				TokenGenFn: func(userID, email, site string) (string, error) {
 					claims := token.Claims{
 						Handshake: &token.Handshake{ID: userID + "::" + email},
@@ -785,7 +838,10 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
 					return tkn, nil
 				},
 			}
-			smtpParams := notify.SmtpParams{
+			if s.Notify.Email.AdminNotifications {
+				emailParams.AdminEmails = s.Admin.Shared.Email
+			}
+			smtpParams := notify.SMTPParams{
 				Host:     s.SMTP.Host,
 				Port:     s.SMTP.Port,
 				TLS:      s.SMTP.TLS,
@@ -805,7 +861,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
 		}
 	}
 
-	if len(destinations) != 0 {
+	if len(destinations) > 0 {
 		log.Printf("[INFO] make notify, types=%s", s.Notify.Type)
 		notifyService = notify.NewService(dataStore, s.Notify.QueueSize, destinations...)
 	}
@@ -833,8 +889,8 @@ func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
 		config.ACMELocation = s.SSL.ACMELocation
 		if s.SSL.ACMEEmail != "" {
 			config.ACMEEmail = s.SSL.ACMEEmail
-		} else if s.Admin.Type == "shared" && s.Admin.Shared.Email != "" {
-			config.ACMEEmail = s.Admin.Shared.Email
+		} else if s.Admin.Type == "shared" && len(s.Admin.Shared.Email) != 0 {
+			config.ACMEEmail = s.Admin.Shared.Email[0]
 		} else if u, e := url.Parse(s.RemarkURL); e == nil {
 			config.ACMEEmail = "admin@" + u.Hostname()
 		}
@@ -842,15 +898,17 @@ func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
 	return config, err
 }
 
-func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Store, admns admin.Store) *auth.Service {
+func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Store, admns admin.Store, authRefreshCache *authRefreshCache) (*auth.Service, error) {
 	authenticator := auth.NewService(auth.Opts{
 		URL:            strings.TrimSuffix(s.RemarkURL, "/"),
 		Issuer:         "remark42",
 		TokenDuration:  s.Auth.TTL.JWT,
 		CookieDuration: s.Auth.TTL.Cookie,
+		SendJWTHeader:  s.Auth.SendJWTHeader,
+		SameSiteCookie: s.parseSameSite(s.Auth.SameSite),
 		SecureCookies:  strings.HasPrefix(s.RemarkURL, "https://"),
-		SecretReader: token.SecretFunc(func() (string, error) { // get secret per site
-			return admns.Key()
+		SecretReader: token.SecretFunc(func(aud string) (string, error) { // get secret per site
+			return admns.Key("")
 		}),
 		ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
 			if c.User == nil {
@@ -863,6 +921,18 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			if err != nil {
 				log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
 			}
+
+			// don't allow anonymous and email with admin's name
+			if strings.HasPrefix(c.User.ID, "anonymous_") || strings.HasPrefix(c.User.ID, "email_") {
+				for _, a := range s.RestrictedNames {
+					if strings.EqualFold(strings.TrimSpace(c.User.Name), a) {
+						c.User.SetBoolAttr("blocked", true)
+						log.Printf("[INFO] blocked %+v, attempt to impersonate (restricted names)", c.User)
+						break
+					}
+				}
+			}
+
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
@@ -880,28 +950,48 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
 		Logger:            log.Default(),
-		RefreshCache:      newAuthRefreshCache(),
+		RefreshCache:      authRefreshCache,
 		UseGravatar:       true,
 	})
-	s.addAuthProviders(authenticator)
-	return authenticator
+
+	if err := s.addAuthProviders(authenticator); err != nil {
+		return nil, err
+	}
+
+	return authenticator, nil
+}
+
+func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
+	switch strings.ToLower(ss) {
+	case "default":
+		return http.SameSiteDefaultMode
+	case "none":
+		return http.SameSiteNoneMode
+	case "lax":
+		return http.SameSiteLaxMode
+	case "strict":
+		return http.SameSiteStrictMode
+	default:
+		return http.SameSiteDefaultMode
+	}
 }
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*authcache.Cache
+	cache.LoadingCache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
+	expirableCache, _ := cache.NewExpirableCache(cache.TTL(5 * time.Minute))
+	return &authRefreshCache{LoadingCache: expirableCache}
 }
 
 // Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
-	return c.Cache.Get(key.(string))
+	return c.LoadingCache.Peek(key.(string))
 }
 
 // Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
+	_, _ = c.LoadingCache.Get(key.(string), func() (interface{}, error) { return value, nil })
 }

backend/app/cmd/server_test.go

@@ -17,7 +17,8 @@ import (
 
 	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+	"go.uber.org/goleak"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -80,10 +81,10 @@ func TestServerApp_DevMode(t *testing.T) {
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, "pong", string(body))
 
 	cancel()
@@ -114,27 +115,99 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as good anonymous
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims := getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+
+	// try to login with non-latin name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	// try to login with bad name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
+	// try to login with name what have space in prefix
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with name what have space in suffix
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with long name
+	ln := strings.Repeat("x", 65)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with admin name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as anonymous with admin name
+	client = http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims = getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	assert.True(t, claims.User.BoolAttr("blocked"), "should be blocked")
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+
 	cancel()
 	app.Wait()
 }
 
+func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (tkn string, claims token.Claims) {
+	var err error
+	for _, c := range resp.Cookies() {
+		if c.Name == "JWT" {
+			tkn = c.Value
+			claims, err = app.restSrv.Authenticator.TokenService().Parse(c.Value)
+			require.NoError(t, err)
+		}
+	}
+	return tkn, claims
+}
+
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
 	sslPort := chooseRandomUnusedPort()
@@ -245,7 +318,7 @@ func TestServerApp_Failed(t *testing.T) {
 	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
 	assert.NoError(t, err)
 	_, err = opts.newServerApp()
-	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
+	assert.EqualError(t, err, "failed to create backup store: can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
 	t.Log(err)
 
 	// invalid url
@@ -268,6 +341,19 @@ func TestServerApp_Failed(t *testing.T) {
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah")
 	t.Log(err)
+
+	// wrong redis location
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+	p = flags.NewParser(&opts, flags.Default)
+	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--cache.type=redis_pub_sub", "--cache.redis_addr=wrong_address"})
+	assert.NoError(t, err)
+	_, err = opts.newServerApp()
+	assert.EqualError(t, err,
+		"failed to make cache: cache backend initialization, redis PubSub initialisation: "+
+			"problem subscribing to channel remark42-cache on address wrong_address: "+
+			"dial tcp: address wrong_address: missing port in address")
+	t.Log(err)
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
@@ -325,6 +411,7 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
 		"--auth.email.user=test_user",
 		"--auth.email.passwd=test_password",
 		"--auth.email.timeout=15s",
+		"--auth.email.template=file.tmpl",
 	}
 	assert.Empty(t, s.SMTP.Host)
 	assert.Empty(t, s.SMTP.Port)
@@ -343,6 +430,7 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
 			{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"},
 			{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"},
 			{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.template", RemoveVersion: "1.9.0"},
 		},
 		deprecatedFlags)
 	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
@@ -433,7 +521,7 @@ func TestServerAuthHooks(t *testing.T) {
 	req.Header.Set("X-JWT", tk)
 	resp, err := client.Do(req)
 	require.NoError(t, err)
-	defer resp.Body.Close()
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
 	// add comment with no-aud claim
@@ -443,15 +531,15 @@ func TestServerAuthHooks(t *testing.T) {
 	require.NoError(t, err)
 	t.Logf("no-aud claims: %s", tkNoAud)
 	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
-		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/", 
-"site": "remark"}}`))
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/",
+	"site": "remark"}}`))
 	require.NoError(t, err)
 	req.Header.Set("X-JWT", tkNoAud)
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
 
 	// block user dev as admin
@@ -461,10 +549,10 @@ func TestServerAuthHooks(t *testing.T) {
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
 	b, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	t.Log(string(b))
 
 	// try add a comment with blocked user
@@ -474,29 +562,51 @@ func TestServerAuthHooks(t *testing.T) {
 	req.Header.Set("X-JWT", tk)
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
 		"blocked user can't post, \n"+tk+"\n"+string(body))
 
 	cancel()
 	app.Wait()
+	client.CloseIdleConnections()
 }
 
 func TestServer_loadEmailTemplate(t *testing.T) {
 	cmd := ServerCommand{}
 	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
-	r := cmd.loadEmailTemplate()
+	r, err := cmd.loadEmailTemplate()
+	assert.NoError(t, err)
 	assert.Equal(t, "The token is {{.Token}}", r)
 
-	cmd.Auth.Email.MsgTemplate = ""
-	r = cmd.loadEmailTemplate()
-	assert.Contains(t, r, "Remark42</h1>")
+	cmd.Auth.Email.MsgTemplate = "badpath.tmpl"
+	r, err = cmd.loadEmailTemplate()
+	assert.EqualError(t, err, "failed to read file badpath.tmpl: open badpath.tmpl: no such file or directory")
+	assert.Equal(t, r, "")
+}
 
-	cmd.Auth.Email.MsgTemplate = "bad-file"
-	r = cmd.loadEmailTemplate()
-	assert.Contains(t, r, "Remark42</h1>")
+func TestServerCommand_parseSameSite(t *testing.T) {
+
+	tbl := []struct {
+		inp string
+		res http.SameSite
+	}{
+		{"", http.SameSiteDefaultMode},
+		{"default", http.SameSiteDefaultMode},
+		{"blah", http.SameSiteDefaultMode},
+		{"none", http.SameSiteNoneMode},
+		{"lax", http.SameSiteLaxMode},
+		{"strict", http.SameSiteStrictMode},
+	}
+
+	cmd := ServerCommand{}
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.res, cmd.parseSameSite(tt.inp))
+		})
+	}
 }
 
 func chooseRandomUnusedPort() (port int) {
@@ -561,11 +671,17 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
 	cmd.SMTP.Password = "test_password"
 	cmd.SMTP.TimeOut = time.Second
 	cmd.UpdateLimit = 10
+	cmd.Admin.Type = "shared"
+	cmd.Admin.Shared.Admins = []string{"id1", "id2"}
+	cmd.RestrictedNames = []string{"umputun", "bobuk"}
 	cmd = fn(cmd)
 
 	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
 
-	// create app
+	return createAppFromCmd(t, cmd)
+}
+
+func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
 	app, err := cmd.newServerApp()
 	require.NoError(t, err)
 
@@ -573,3 +689,8 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
 	rand.Seed(time.Now().UnixNano())
 	return app, ctx, cancel
 }
+
+func TestMain(m *testing.M) {
+	// ignore is added only for GitHub Actions, can't reproduce locally
+	goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
+}

backend/app/main.go

@@ -8,9 +8,9 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
-	"github.com/umputun/remark/backend/app/cmd"
+	"github.com/umputun/remark42/backend/app/cmd"
 )
 
 // Opts with all cli commands and flags
@@ -84,6 +84,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
+// nolint:gochecknoinits // can't avoid it in this place
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal)

backend/app/main_test.go

@@ -13,9 +13,9 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 )
 
 func Test_Main(t *testing.T) {
@@ -31,8 +31,8 @@ func Test_Main(t *testing.T) {
 	done := make(chan struct{})
 	go func() {
 		<-done
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.NoError(t, err)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.NoError(t, e)
 	}()
 
 	finished := make(chan struct{})
@@ -62,7 +62,7 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "goroutine"))
 	assert.True(t, strings.Contains(dump, "[running]"))
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
-	log.Printf("\n dump: %s", dump)
+	t.Logf("\n dump: %s", dump)
 }
 
 func chooseRandomUnusedPort() (port int) {
@@ -87,3 +87,12 @@ func waitForHTTPServerStart(port int) {
 		}
 	}
 }
+
+func TestMain(m *testing.M) {
+	// both ignores are for leaks which are detected locally
+	goleak.VerifyTestMain(
+		m,
+		goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
+		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+	)
+}

backend/app/migrator/backup_test.go

@@ -77,7 +77,7 @@ func TestBackup_Do(t *testing.T) {
 
 type mockExporter struct{}
 
-func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
+func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
 	_, err := w.Write([]byte("some export blah blah 1234567890"))
 	return 1000, err
 }

backend/app/migrator/disqus.go

@@ -9,7 +9,7 @@ import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // Disqus implements Importer from disqus xml
@@ -52,9 +52,8 @@ type uid struct {
 
 // Import from disqus and save to store
 func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
-
-	if err = d.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := d.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
 	commentsCh := d.convert(r, siteID)
@@ -136,9 +135,10 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 						Text:      d.cleanText(comment.Message),
 						Timestamp: comment.CreatedAt,
 						ParentID:  comment.Pid.Val,
+						Imported:  true,
 					}
-					if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
-						c.User.ID = "disqus_" + c.User.Name
+					if comment.AuthorUserName == "" { // empty comment.AuthorUserName from disqus
+						c.User.ID = "disqus_" + store.EncodeID(c.User.Name)
 					}
 					if c.ID == "" { // no comment.UID
 						c.ID = comment.ID

backend/app/migrator/disqus_test.go

@@ -6,14 +6,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestDisqus_Import(t *testing.T) {
@@ -39,6 +39,11 @@ func TestDisqus_Import(t *testing.T) {
 	assert.Equal(t, "Alexander Blah", c.User.Name)
 	assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID)
 	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
+	assert.True(t, c.Imported)
+
+	c = last[1] // get comment with empty username
+	assert.Equal(t, "No Username", c.User.Name)
+	assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
 
 	posts, err := dataStore.List("test", 0, 0)
 	assert.NoError(t, err)
@@ -71,6 +76,7 @@ func TestDisqus_Convert(t *testing.T) {
 			ID:   "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434",
 			IP:   "178.178.178.178",
 		},
+		Imported: true,
 	}
 	exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z")
 	assert.Equal(t, exp0, res[0])
@@ -169,10 +175,9 @@ var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
 		<isDeleted>false</isDeleted>
 		<isSpam>false</isSpam>
 		<author>
-			<email>dmitri.noname@gmail.com</email>
-			<name>Dmitry Noname</name>
+			<email>john.nousername@gmail.com</email>
+			<name>No Username</name>
 			<isAnonymous>false</isAnonymous>
-			<username>google-74b9e7568ef6860e93862c5d77590123</username>
 		</author>
 		<ipAddress>89.89.89.139</ipAddress>
 		<thread dsq:id="247918464"/>

backend/app/migrator/mapper.go

@@ -7,15 +7,15 @@ import (
 	"strings"
 )
 
-// UrlMapper implements Mapper interface
-type UrlMapper struct {
+// URLMapper implements Mapper interface
+type URLMapper struct {
 	rules map[string]string
 }
 
-// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
+// NewURLMapper reads rules from given reader and returns initialized URLMapper
 // if given rules are valid.
-func NewUrlMapper(reader io.Reader) (Mapper, error) {
-	u := &UrlMapper{}
+func NewURLMapper(reader io.Reader) (Mapper, error) {
+	u := &URLMapper{}
 	if err := u.loadRules(reader); err != nil {
 		return u, err
 	}
@@ -29,7 +29,7 @@ func NewUrlMapper(reader io.Reader) (Mapper, error) {
 // Example:
 // https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
 // https://www.myblog.com/* https://myblog.com/*
-func (u *UrlMapper) loadRules(reader io.Reader) error {
+func (u *URLMapper) loadRules(reader io.Reader) error {
 	data, err := ioutil.ReadAll(reader)
 	if err != nil {
 		return err
@@ -54,19 +54,19 @@ func (u *UrlMapper) loadRules(reader io.Reader) error {
 
 // URL maps given url to another url according loaded url-rules.
 // If not matched returns given url.
-func (u *UrlMapper) URL(url string) string {
-	if newUrl, ok := u.rules[url]; ok {
-		return newUrl
+func (u *URLMapper) URL(url string) string {
+	if newURL, ok := u.rules[url]; ok {
+		return newURL
 	}
 	// try to match by prefix
-	for oldUrl, newUrl := range u.rules {
-		if !strings.HasSuffix(oldUrl, "*") {
+	for oldURL, newURL := range u.rules {
+		if !strings.HasSuffix(oldURL, "*") {
 			continue
 		}
-		oldUrl = strings.TrimSuffix(oldUrl, "*")
-		newUrl = strings.TrimSuffix(newUrl, "*")
-		if strings.HasPrefix(url, oldUrl) {
-			return newUrl + strings.TrimPrefix(url, oldUrl)
+		oldURL = strings.TrimSuffix(oldURL, "*")
+		newURL = strings.TrimSuffix(newURL, "*")
+		if strings.HasPrefix(url, oldURL) {
+			return newURL + strings.TrimPrefix(url, oldURL)
 		}
 	}
 	// search failed, return given url

backend/app/migrator/mapper_test.go

@@ -16,7 +16,7 @@ https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/
 https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
 `)
 
-	mapper, err := NewUrlMapper(rules)
+	mapper, err := NewURLMapper(rules)
 	assert.NoError(t, err)
 
 	// if url not matched mapper should return given url
@@ -30,7 +30,7 @@ https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.c
 
 	// want remap from http to https
 	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
-	mapper, err = NewUrlMapper(rules)
+	mapper, err = NewURLMapper(rules)
 	assert.NoError(t, err)
 	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
 	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
@@ -38,7 +38,7 @@ https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.c
 
 	// want remap from http to https by pattern
 	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
-	mapper, err = NewUrlMapper(rules)
+	mapper, err = NewURLMapper(rules)
 	assert.NoError(t, err)
 	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
 	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
@@ -80,7 +80,7 @@ func TestUrlMapper_New(t *testing.T) {
 		},
 	}
 	for _, c := range cases {
-		_, err := NewUrlMapper(strings.NewReader(c.rules))
+		_, err := NewURLMapper(strings.NewReader(c.rules))
 		if c.expectError {
 			assert.Error(t, err)
 		} else {

backend/app/migrator/migrator.go

@@ -10,8 +10,8 @@ import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 // Importer defines interface to convert posts from external sources

backend/app/migrator/migrator_test.go

@@ -6,14 +6,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestMigrator_ImportDisqus(t *testing.T) {

backend/app/migrator/native.go

@@ -11,8 +11,8 @@ import (
 	"github.com/go-pkgz/syncs"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 const nativeVersion = 1
@@ -140,8 +140,8 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
-	if err = n.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := n.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
 	var failed, total, comments int64
@@ -155,6 +155,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 	for {
 		comment := store.Comment{}
 		err = dec.Decode(&comment)
+		comment.Imported = true
 		if err == io.EOF {
 			break
 		}

backend/app/migrator/native_test.go

@@ -10,15 +10,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestNative_Export(t *testing.T) {
@@ -35,7 +34,7 @@ func TestNative_Export(t *testing.T) {
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
-	log.Print(c1)
+	t.Log(c1)
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
@@ -74,7 +73,7 @@ func TestNative_Import(t *testing.T) {
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
-	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00","imported":false}`
 
 	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
 	r := Native{DataStore: b}
@@ -88,10 +87,12 @@ func TestNative_Import(t *testing.T) {
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
 	assert.Equal(t, "1234", comments[0].ParentID)
 	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.True(t, comments[0].Imported)
 
 	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
 	assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
 	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.True(t, comments[1].Imported)
 
 	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
 	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
@@ -106,7 +107,7 @@ func TestNative_ImportWithMapper(t *testing.T) {
 
 	// want to remap comments to https://rdt.c
 	rules := `https://radio-t.com* https://rdt.c*`
-	mapper, err := NewUrlMapper(strings.NewReader(rules))
+	mapper, err := NewURLMapper(strings.NewReader(rules))
 	assert.NoError(t, err)
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
@@ -179,11 +180,11 @@ func TestNative_ImportManyWithError(t *testing.T) {
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) (*service.DataStore, func()) {
+func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
 
-	testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
+	testDB := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
 
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
+	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDB})
 	assert.NoError(t, err)
 
 	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
@@ -208,6 +209,6 @@ func prep(t *testing.T) (*service.DataStore, func()) {
 
 	return b, func() {
 		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
+		_ = os.Remove(testDB)
 	}
 }

backend/app/migrator/wordpress.go

@@ -9,7 +9,7 @@ import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 const wpTimeLayout = "2006-01-02 15:04:05"
@@ -61,8 +61,8 @@ func (w *WordPress) Convert(text string) string {
 // Import comments from WP and save to store
 func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
 
-	if err = w.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := w.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
 	commentsCh := w.convert(r, siteID)
@@ -139,6 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								Text:      comment.Content,
 								Timestamp: comment.Date.time,
 								ParentID:  comment.PID,
+								Imported:  true,
 							}
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++

backend/app/migrator/wordpress_test.go

@@ -6,14 +6,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestWordPress_Import(t *testing.T) {
@@ -42,6 +42,7 @@ func TestWordPress_Import(t *testing.T) {
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
 	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
 	assert.NoError(t, err)
@@ -77,6 +78,7 @@ func TestWordPress_Convert(t *testing.T) {
 			ID:   "wordpress_" + store.EncodeID("Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual"),
 			IP:   "74.200.244.101",
 		},
+		Imported: true,
 	}
 	exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
 	assert.Equal(t, exp1, comments[1])
@@ -190,7 +192,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/notify/email.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"mime"
 	"mime/quotedprintable"
 	"net"
 	"net/smtp"
@@ -14,23 +15,27 @@ import (
 
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/templates"
 )
 
 // EmailParams contain settings for email notifications
 type EmailParams struct {
-	From                 string // from email address
-	MsgTemplate          string // request message template
-	VerificationSubject  string // verification message subject
-	VerificationTemplate string // verification message template
-	SubscribeURL         string // full subscribe handler URL
-	UnsubscribeURL       string // full unsubscribe handler URL
+	From                     string   // from email address
+	AdminEmails              []string // administrator emails to send copy of comment notification to
+	MsgTemplatePath          string   // path to request message template
+	VerificationSubject      string   // verification message sub
+	VerificationTemplatePath string   // path to verification template
+	SubscribeURL             string   // full subscribe handler URL
+	UnsubscribeURL           string   // full unsubscribe handler URL
 
 	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
 }
 
-// SmtpParams contain settings for smtp server connection
-type SmtpParams struct {
+// SMTPParams contain settings for smtp server connection
+type SMTPParams struct {
 	Host     string        // SMTP host
 	Port     int           // SMTP port
 	TLS      bool          // TLS auth
@@ -42,7 +47,7 @@ type SmtpParams struct {
 // Email implements notify.Destination for email
 type Email struct {
 	EmailParams
-	SmtpParams
+	SMTPParams
 
 	smtp       smtpClientCreator
 	msgTmpl    *template.Template // parsed request message template
@@ -64,7 +69,7 @@ type smtpClient interface {
 
 // smtpClientCreator interface defines function for creating new smtpClients
 type smtpClientCreator interface {
-	Create(SmtpParams) (smtpClient, error)
+	Create(SMTPParams) (smtpClient, error)
 }
 
 type emailMessage struct {
@@ -88,6 +93,7 @@ type msgTmplData struct {
 	PostTitle         string
 	Email             string
 	UnsubscribeLink   string
+	ForAdmin          bool
 }
 
 // verifyTmplData store data for verification message template execution
@@ -100,171 +106,123 @@ type verifyTmplData struct {
 }
 
 const (
-	defaultVerificationSubject = "Email verification"
-	defaultEmailTimeout        = 10 * time.Second
-	defaultEmailTemplate       = `<!DOCTYPE html>
-<html>
-<head>
-	<meta name="viewport" content="width=device-width" />
-	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-	<style type="text/css">
-		img {
-			max-width: 100%;
-			max-height: 250px;
-			margin: 5px 0;
-			display: block;
-			color: #000;
-		}
-		a {
-			text-decoration: none;
-			color: #0aa;
-		}
-		p {
-			margin: 0 0 12px;
-		}
-	</style>
-</head>
-<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
-<body>
-	<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
-		<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
-		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
-		<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
-			<div>
-				<div style="margin-bottom: 12px; line-height: 24px;">
-					<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
-					<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
-					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
-					<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
-				</div>
-				<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
-					{{.ParentCommentText}}
-				</div>
-			</div>
-			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px; line-height: 24px;">
-				<div style="margin-bottom: 8px;">
-					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
-					<div style="float: left; font-size: 14px; font-weight: bold; color: #777">
-						{{.UserName}}
-					</div>
-					<div style="color: #999; font-size: 14px; margin: 0 8px; float: left;">
-						{{.CommentDate.Format "02.01.2006 at 15:04"}}
-					</div>
-					<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
-				</div>
-				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">
-					{{.CommentText}}
-				</div>
-			</div>
-		</div>
-		<div style="text-align: center; font-size: 14px; margin-top: 32px;">
-			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a> for {{.ParentUserName}}</i>
-			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;">
-				<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
-				<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
-				<div style="opacity: 0;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
-			</div>
-		</div>
-	</div>
-</body>
-</html>
-`
-	defaultEmailVerificationTemplate = `<!DOCTYPE html>
-<html>
-<head>
-	<meta name="viewport" content="width=device-width" />
-	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-</head>
-<body>
-	<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
-	<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
-		<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
-		<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
-		{{if .SubscribeURL}}
-		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
-		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
-		{{ end }}
-		<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
-			<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
-			<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
-			<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
-		</div>
-		<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
-	</div>
-</body>
-</html>
-`
+	defaultVerificationSubject           = "Email verification"
+	defaultEmailTimeout                  = 10 * time.Second
+	defaultEmailTemplatePath             = "email_reply.html.tmpl"
+	defaultEmailVerificationTemplatePath = "email_confirmation_subscription.html.tmpl"
 )
 
 // NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
-func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
+func NewEmail(emailParams EmailParams, smtpParams SMTPParams) (*Email, error) {
 	// set up Email emailParams
 	res := Email{EmailParams: emailParams}
-	if res.MsgTemplate == "" {
-		res.MsgTemplate = defaultEmailTemplate
-	}
-	if res.VerificationTemplate == "" {
-		res.VerificationTemplate = defaultEmailVerificationTemplate
+	res.smtp = &emailClient{}
+	res.SMTPParams = smtpParams
+	if res.TimeOut <= 0 {
+		res.TimeOut = defaultEmailTimeout
 	}
+
 	if res.VerificationSubject == "" {
 		res.VerificationSubject = defaultVerificationSubject
 	}
 
-	// set up SMTP emailParams
-	res.smtp = &emailClient{}
-	res.SmtpParams = smtpParams
-	if res.TimeOut <= 0 {
-		res.TimeOut = defaultEmailTimeout
+	// initialize templates
+	err := res.setTemplates()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't set templates")
 	}
 
 	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
 		res.Host, res.Username, res.TimeOut)
 
-	// initialise templates
-	var err error
-	if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
-		return nil, errors.Wrapf(err, "can't parse message template")
-	}
-	if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
-		return nil, errors.Wrapf(err, "can't parse verification template")
-	}
-	return &res, err
+	return &res, nil
 }
 
-// Send email about reply to Request.Email if it's set, otherwise do nothing and return nil, thread safe
-// do not returns sending error, only following:
-// 1. (likely impossible) template execution error from email message creation from Request
-// 2. message dropped without sending in case of closed ctx
-func (e *Email) Send(ctx context.Context, req Request) (err error) {
+func (e *Email) setTemplates() error {
+	var err error
+	var msgTmplFile, verifyTmplFile []byte
+	fs := templates.NewFS()
+
+	if e.VerificationTemplatePath == "" {
+		e.VerificationTemplatePath = defaultEmailVerificationTemplatePath
+	}
+
+	if e.MsgTemplatePath == "" {
+		e.MsgTemplatePath = defaultEmailTemplatePath
+	}
+
+	if msgTmplFile, err = fs.ReadFile(e.MsgTemplatePath); err != nil {
+		return errors.Wrapf(err, "can't read message template")
+	}
+	if verifyTmplFile, err = fs.ReadFile(e.VerificationTemplatePath); err != nil {
+		return errors.Wrapf(err, "can't read verification template")
+	}
+	if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil {
+		return errors.Wrapf(err, "can't parse message template")
+	}
+	if e.verifyTmpl, err = template.New("verifyTmpl").Parse(string(verifyTmplFile)); err != nil {
+		return errors.Wrapf(err, "can't parse verification template")
+	}
+
+	return nil
+}
+
+// Send email about comment reply to Request.Emails and Email.AdminEmails
+// if they're set.
+// Thread safe
+func (e *Email) Send(ctx context.Context, req Request) error {
+	select {
+	case <-ctx.Done():
+		return errors.Errorf("sending email messages about comment %q aborted due to canceled context", req.Comment.ID)
+	default:
+	}
+
+	result := new(multierror.Error)
+
+	for _, email := range req.Emails {
+		err := e.buildAndSendMessage(ctx, req, email, false)
+		result = multierror.Append(errors.Wrapf(err, "problem sending user email notification to %q", email))
+	}
+
+	for _, email := range e.AdminEmails {
+		err := e.buildAndSendMessage(ctx, req, email, true)
+		result = multierror.Append(errors.Wrapf(err, "problem sending admin email notification to %q", email))
+	}
+
+	return result.ErrorOrNil()
+}
+
+func (e *Email) buildAndSendMessage(ctx context.Context, req Request, email string, forAdmin bool) error {
+	log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
+	msg, err := e.buildMessageFromRequest(req, email, forAdmin)
+	if err != nil {
+		return err
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.sendMessage(emailMessage{from: e.From, to: email, message: msg})
+		})
+}
+
+// SendVerification email verification VerificationRequest.Email if it's set.
+// Thread safe
+func (e *Email) SendVerification(ctx context.Context, req VerificationRequest) error {
 	if req.Email == "" {
 		// this means we can't send this request via Email
 		return nil
 	}
 	select {
 	case <-ctx.Done():
-		return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
+		return errors.Errorf("sending message to %q aborted due to canceled context", req.User)
 	default:
 	}
-	var msg string
 
-	if req.Verification.Token != "" {
-		log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
-		msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
-		if err != nil {
-			return err
-		}
-	}
-
-	if req.Comment.ID != "" {
-		if req.parent.User.ID == req.Comment.User.ID {
-			// don't send anything if if user replied to their own comment
-			return nil
-		}
-		log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
-		msg, err = e.buildMessageFromRequest(req)
-		if err != nil {
-			return err
-		}
+	log.Printf("[DEBUG] send verification via %s, user %s", e, req.User)
+	msg, err := e.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
+	if err != nil {
+		return err
 	}
 
 	return repeater.NewDefault(5, time.Millisecond*250).Do(
@@ -292,37 +250,50 @@ func (e *Email) buildVerificationMessage(user, email, token, site string) (strin
 }
 
 // buildMessageFromRequest generates email message based on Request using e.MsgTemplate
-func (e *Email) buildMessageFromRequest(req Request) (string, error) {
+func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool) (string, error) {
 	subject := "New reply to your comment"
-	if req.Comment.PostTitle != "" {
-		subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
+	if forAdmin {
+		subject = "New comment to your site"
 	}
-	token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
-	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if req.Comment.PostTitle != "" {
+		subject += fmt.Sprintf(" for %q", req.Comment.PostTitle)
+	}
+
+	token, err := e.TokenGenFn(req.parent.User.ID, email, req.Comment.Locator.SiteID)
 	if err != nil {
 		return "", errors.Wrapf(err, "error creating token for unsubscribe link")
 	}
-	commentUrlPrefix := req.Comment.Locator.URL + uiNav
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if forAdmin {
+		unsubscribeLink = ""
+	}
+
+	commentURLPrefix := req.Comment.Locator.URL + uiNav
 	msg := bytes.Buffer{}
-	err = e.msgTmpl.Execute(&msg, msgTmplData{
-		UserName:          req.Comment.User.Name,
-		UserPicture:       req.Comment.User.Picture,
-		CommentText:       req.Comment.Text,
-		CommentLink:       commentUrlPrefix + req.Comment.ID,
-		CommentDate:       req.Comment.Timestamp,
-		ParentUserName:    req.parent.User.Name,
-		ParentUserPicture: req.parent.User.Picture,
-		ParentCommentText: req.parent.Text,
-		ParentCommentLink: commentUrlPrefix + req.parent.ID,
-		ParentCommentDate: req.parent.Timestamp,
-		PostTitle:         req.Comment.PostTitle,
-		Email:             req.Email,
-		UnsubscribeLink:   unsubscribeLink,
-	})
+	tmplData := msgTmplData{
+		UserName:        req.Comment.User.Name,
+		UserPicture:     req.Comment.User.Picture,
+		CommentText:     req.Comment.Text,
+		CommentLink:     commentURLPrefix + req.Comment.ID,
+		CommentDate:     req.Comment.Timestamp,
+		PostTitle:       req.Comment.PostTitle,
+		Email:           email,
+		UnsubscribeLink: unsubscribeLink,
+		ForAdmin:        forAdmin,
+	}
+	// in case of message to admin, parent message might be empty
+	if req.Comment.ParentID != "" {
+		tmplData.ParentUserName = req.parent.User.Name
+		tmplData.ParentUserPicture = req.parent.User.Picture
+		tmplData.ParentCommentText = req.parent.Text
+		tmplData.ParentCommentLink = commentURLPrefix + req.parent.ID
+		tmplData.ParentCommentDate = req.parent.Timestamp
+	}
+	err = e.msgTmpl.Execute(&msg, tmplData)
 	if err != nil {
 		return "", errors.Wrapf(err, "error executing template to build comment reply message")
 	}
-	return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
+	return e.buildMessage(subject, msg.String(), email, "text/html", unsubscribeLink)
 }
 
 // buildMessage generates email message to send using net/smtp.Data()
@@ -333,7 +304,7 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
 	}
 	message = addHeader(message, "From", e.From)
 	message = addHeader(message, "To", to)
-	message = addHeader(message, "Subject", subject)
+	message = addHeader(message, "Subject", mime.BEncoding.Encode("utf-8", subject))
 	message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
 
 	if contentType != "" {
@@ -354,7 +325,10 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
 	if _, err := qp.Write([]byte(body)); err != nil {
 		return "", err
 	}
-	defer qp.Close()
+	// flush now, must NOT use defer, for small body, defer may cause buff.String() got empty body
+	if err := qp.Close(); err != nil {
+		return "", fmt.Errorf("quotedprintable Write failed: %w", err)
+	}
 	m := buff.String()
 	message += "\n" + m
 	return message, nil
@@ -364,30 +338,30 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
 // Thread safe.
 func (e *Email) sendMessage(m emailMessage) error {
 	if e.smtp == nil {
-		return errors.New("sendMessage called without smtpClient set")
+		return errors.New("sendMessage called without client set")
 	}
-	smtpClient, err := e.smtp.Create(e.SmtpParams)
+	client, err := e.smtp.Create(e.SMTPParams)
 	if err != nil {
 		return errors.Wrap(err, "failed to make smtp Create")
 	}
 
 	defer func() {
-		if err := smtpClient.Quit(); err != nil {
+		if err = client.Quit(); err != nil {
 			log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
-			if err := smtpClient.Close(); err != nil {
+			if err = client.Close(); err != nil {
 				log.Printf("[WARN] can't close smtp connection, %v", err)
 			}
 		}
 	}()
 
-	if err := smtpClient.Mail(m.from); err != nil {
+	if err = client.Mail(m.from); err != nil {
 		return errors.Wrapf(err, "bad from address %q", m.from)
 	}
-	if err := smtpClient.Rcpt(m.to); err != nil {
+	if err = client.Rcpt(m.to); err != nil {
 		return errors.Wrapf(err, "bad to address %q", m.to)
 	}
 
-	writer, err := smtpClient.Data()
+	writer, err := client.Data()
 	if err != nil {
 		return errors.Wrap(err, "can't make email writer")
 	}
@@ -411,9 +385,9 @@ func (e *Email) String() string {
 	return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
 }
 
-// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SmtpParams
+// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SMTPParams
 // and returns pointer to it. Thread safe.
-func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
+func (s *emailClient) Create(params SMTPParams) (smtpClient, error) {
 	authenticate := func(c *smtp.Client) error {
 		if params.Username == "" || params.Password == "" {
 			return nil
@@ -431,6 +405,7 @@ func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
 		tlsConf := &tls.Config{
 			InsecureSkipVerify: false,
 			ServerName:         params.Host,
+			MinVersion:         tls.VersionTLS12,
 		}
 		conn, err := tls.Dial("tcp", srvAddress, tlsConf)
 		if err != nil {
@@ -447,7 +422,7 @@ func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
 		return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
 	}
 
-	c, err = smtp.NewClient(conn, srvAddress)
+	c, err = smtp.NewClient(conn, params.Host)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to dial")
 	}

backend/app/notify/email_test.go

@@ -12,83 +12,92 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestEmailNew(t *testing.T) {
-	var testSet = []struct {
+	emailParams := EmailParams{
+		From:                     "test@from",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}
+	smtpParams := SMTPParams{
+		Host:     "test@host",
+		Port:     1000,
+		TLS:      true,
+		Username: "test@username",
+		Password: "test@password",
+		TimeOut:  time.Second,
+	}
+
+	email, err := NewEmail(emailParams, smtpParams)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "email returned")
+
+	assert.NotNil(t, email.msgTmpl, "e.template is set")
+	assert.Equal(t, emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
+	if smtpParams.TimeOut == 0 {
+		assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
+	} else {
+		assert.Equal(t, smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
+	}
+	assert.Equal(t, smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
+	assert.Equal(t, smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
+	assert.Equal(t, smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
+	assert.Equal(t, smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
+	assert.Equal(t, smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
+}
+
+func Test_initTemplatesErr(t *testing.T) {
+	testSet := []struct {
 		name        string
-		err         bool
 		errText     string
 		emailParams EmailParams
-		smtpParams  SmtpParams
 	}{
-		{name: "empty"},
-		{name: "with template parse error",
-			err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
+		{
+			name:    "with wrong path to verification template",
+			errText: "can't read verification template: open notfount.tmpl: no such file or directory",
 			emailParams: EmailParams{
-				MsgTemplate: "{{",
-			}},
-		{name: "with verification template parse error",
-			err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
-			emailParams: EmailParams{
-				From:                 "test@from",
-				VerificationTemplate: "{{",
-			},
-			smtpParams: SmtpParams{
-				Host:     "test@host",
-				Port:     1000,
-				TLS:      true,
-				Username: "test@username",
-				Password: "test@password",
-				TimeOut:  time.Second,
+				VerificationTemplatePath: "notfount.tmpl",
+				MsgTemplatePath:          "testdata/msg.html.tmpl",
 			},
 		},
-		{name: "normal creation",
-			err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+		{
+			name:    "with wrong path to message template",
+			errText: "can't read message template: open notfount.tmpl: no such file or directory",
 			emailParams: EmailParams{
-				From: "test@from",
+				VerificationTemplatePath: "testdata/verification.html.tmpl",
+				MsgTemplatePath:          "notfount.tmpl",
 			},
-			smtpParams: SmtpParams{
-				Host:     "test@host",
-				Port:     1000,
-				TLS:      true,
-				Username: "test@username",
-				Password: "test@password",
-				TimeOut:  time.Second,
+		},
+		{
+			name:    "with error on read verification template",
+			errText: "can't parse verification template: template: verifyTmpl",
+			emailParams: EmailParams{
+				VerificationTemplatePath: "testdata/bad.html.tmpl",
+				MsgTemplatePath:          "testdata/msg.html.tmpl",
+			},
+		},
+		{
+			name:    "with error on read message template",
+			errText: "can't parse message template: template: msgTmpl",
+			emailParams: EmailParams{
+				VerificationTemplatePath: "testdata/verification.html.tmpl",
+				MsgTemplatePath:          "testdata/bad.html.tmpl",
 			},
 		},
 	}
+
 	for _, d := range testSet {
+		d := d
 		t.Run(d.name, func(t *testing.T) {
-			email, err := NewEmail(d.emailParams, d.smtpParams)
-
-			if d.err && d.errText == "" {
-				assert.Error(t, err)
-				assert.Nil(t, email)
-			} else if d.err && d.errText != "" {
-				assert.EqualError(t, err, d.errText)
-				assert.Nil(t, email)
-			} else {
-				assert.NoError(t, err)
-				assert.NotNil(t, email, "email returned")
-
-				assert.NotNil(t, email.msgTmpl, "e.template is set")
-				assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
-				assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
-				assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
-				if d.smtpParams.TimeOut == 0 {
-					assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
-				} else {
-					assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
-				}
-				assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
-				assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
-				assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
-				assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
-				assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
-			}
+			e := Email{EmailParams: d.emailParams}
+			err := e.setTemplates()
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), d.errText)
 		})
 	}
 }
@@ -100,41 +109,39 @@ func TestEmailSendErrors(t *testing.T) {
 
 	e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
 	assert.NoError(t, err)
-	assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
+	assert.EqualError(t, e.SendVerification(context.Background(), VerificationRequest{Email: "bad@example.org", Token: "some"}),
 		"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
-	e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
-	assert.NoError(t, err)
 
 	e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
 	assert.NoError(t, err)
-	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
-		"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
-	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
-	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
+		"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\": "+
+			"error executing template to build comment reply message: "+
+			"template: test:1:2: executing \"test\" at <.Test>: "+
+			"can't evaluate field Test in type notify.msgTmplData\n\n")
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
-		"sending message to \"bad@example.org\" aborted due to canceled context")
+	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
+		"sending email messages about comment \"999\" aborted due to canceled context")
 
 	e.smtp = &fakeTestSMTP{}
-	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
-		"error creating token for unsubscribe link: token generation error")
-	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
-	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad@example.org"}}),
+		"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\":"+
+			" error creating token for unsubscribe link: token generation error\n\n")
 }
 
 func TestEmailSend_ExitConditions(t *testing.T) {
-	email, err := NewEmail(EmailParams{}, SmtpParams{})
+	email, err := NewEmail(EmailParams{
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email, "expecting email returned")
 	// prevent triggering e.autoFlush creation
 	emptyRequest := Request{Comment: store.Comment{ID: "999"}}
 	assert.NoError(t, email.Send(context.Background(), emptyRequest),
-		"Message without parent comment User.Email is not sent and returns nil")
-	requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
-	assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
-		"Message with parent comment User equals comment User is not sent and returns nil")
+		"Message without Emails and AdminEmails is not sent and returns nil")
 }
 
 func TestEmailSendClientError(t *testing.T) {
@@ -152,6 +159,7 @@ func TestEmailSendClientError(t *testing.T) {
 			err: "can't make email writer: failed to send"},
 	}
 	for _, d := range testSet {
+		d := d
 		t.Run(d.name, func(t *testing.T) {
 			e := Email{smtp: d.smtp}
 			if d.err != "" {
@@ -178,25 +186,38 @@ func TestEmailSendClientError(t *testing.T) {
 		"e.send called without smtpClient set returns error")
 }
 
+func TestEmail_DefaultTemplates(t *testing.T) {
+	email, err := NewEmail(EmailParams{}, SMTPParams{})
+	assert.Error(t, err)
+	assert.Nil(t, email)
+	email, err = NewEmail(EmailParams{VerificationTemplatePath: "testdata/verification.html.tmpl"}, SMTPParams{})
+	assert.Error(t, err)
+	assert.Nil(t, email)
+}
+
 func TestEmail_Send(t *testing.T) {
-	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	email, err := NewEmail(EmailParams{
+		From:                     "from@example.org",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email)
-	fakeSmtp := fakeTestSMTP{}
-	email.smtp = &fakeSmtp
+	fakeSMTP := fakeTestSMTP{}
+	email.smtp = &fakeSMTP
 	email.TokenGenFn = TokenGenFn
 	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
 	req := Request{
-		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
 		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
-		Email:   "test@example.org",
+		Emails:  []string{"test@example.org"},
 	}
 	assert.NoError(t, email.Send(context.TODO(), req))
-	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
-	assert.Equal(t, 1, fakeSmtp.readQuitCount())
-	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
 	// test buildMessageFromRequest separately for message text
-	res, err := email.buildMessageFromRequest(req)
+	res, err := email.buildMessageFromRequest(req, req.Emails[0], false)
 	assert.NoError(t, err)
 	assert.Contains(t, res, `From: from@example.org
 To: test@example.org
@@ -206,30 +227,97 @@ MIME-version: 1.0
 Content-Type: text/html; charset="UTF-8"
 List-Unsubscribe-Post: List-Unsubscribe=One-Click
 List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
+Date: `)
+
+	// send email to both user and admin, without parent set
+	email.AdminEmails = []string{"admin@example.org"}
+	req = Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Emails:  []string{"test@example.org"},
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
+	assert.Equal(t, 3, fakeSMTP.readQuitCount(), "plus two emails: one for user and one for admin")
+	assert.Equal(t, "admin@example.org", fakeSMTP.readRcpt())
+	res, err = email.buildMessageFromRequest(req, email.AdminEmails[0], true)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: admin@example.org
+Subject: New comment to your site for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+}
+
+func TestEmail_SendWithUnicodeInSubject(t *testing.T) {
+	email, err := NewEmail(EmailParams{
+		From:                     "from@example.org",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSMTP := fakeTestSMTP{}
+	email.smtp = &fakeSMTP
+	email.TokenGenFn = TokenGenFn
+	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
+	req := Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "Привет"},
+		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
+		Emails:  []string{"test@example.org"},
+	}
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildMessageFromRequest(req, req.Emails[0], false)
+	assert.NoError(t, err)
+	// `=?utf-8?b?TmV3IHJlcGx5IHRvIHlvdXIgY29tbWVudCBmb3IgItCf0YDQuNCy0LXRgiI=?=` -> `New reply to your comment for "Привет"` in base64 + required prefix and suffix
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: =?utf-8?b?TmV3IHJlcGx5IHRvIHlvdXIgY29tbWVudCBmb3IgItCf0YDQuNCy0LXRgiI=?=
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
 Date: `)
 }
 
 func TestEmail_SendVerification(t *testing.T) {
-	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	email, err := NewEmail(EmailParams{
+		From:                     "from@example.org",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, SMTPParams{})
 	assert.NoError(t, err)
 	assert.NotNil(t, email)
-	fakeSmtp := fakeTestSMTP{}
-	email.smtp = &fakeSmtp
+	fakeSMTP := fakeTestSMTP{}
+	email.smtp = &fakeSMTP
 	email.TokenGenFn = TokenGenFn
-	req := Request{
-		Email: "test@example.org",
-		Verification: VerificationMetadata{
-			SiteID: "remark",
-			User:   "test_username",
-			Token:  "secret_",
-		},
+	// proper VerificationRequest without email
+	req := VerificationRequest{
+		SiteID: "remark",
+		User:   "test_username",
+		Token:  "secret_",
 	}
-	assert.NoError(t, email.Send(context.TODO(), req))
-	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
-	assert.Equal(t, 1, fakeSmtp.readQuitCount())
-	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
-	// test buildMessageFromRequest separately for message text
-	res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, email.SendVerification(context.TODO(), req))
+	assert.Equal(t, "", fakeSMTP.readMail())
+	assert.Equal(t, 0, fakeSMTP.readQuitCount())
+	assert.Equal(t, "", fakeSMTP.readRcpt())
+
+	// proper VerificationRequest with email
+	req.Email = "test@example.org"
+	assert.NoError(t, email.SendVerification(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
+
+	// VerificationRequest with canceled context
+	ctx, cancel := context.WithCancel(context.TODO())
+	cancel()
+	assert.EqualError(t, email.SendVerification(ctx, req), "sending message to \"test_username\" aborted due to canceled context")
+
+	// test buildVerificationMessage separately for message text
+	res, err := email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
 	assert.NoError(t, err)
 	assert.Contains(t, res, `From: from@example.org
 To: test@example.org
@@ -241,7 +329,7 @@ Date: `)
 	assert.Contains(t, res, `secret_`)
 	assert.NotContains(t, res, `https://example.org/`)
 	email.SubscribeURL = "https://example.org/subscribe.html?token="
-	res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	res, err = email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
 	assert.NoError(t, err)
 	assert.Contains(t, res, `From: from@example.org
 To: test@example.org
@@ -255,7 +343,7 @@ Date: `)
 
 func Test_emailClient_Create(t *testing.T) {
 	creator := emailClient{}
-	client, err := creator.Create(SmtpParams{})
+	client, err := creator.Create(SMTPParams{})
 	assert.Error(t, err, "absence of address to connect results in error")
 	assert.Nil(t, client, "no client returned in case of error")
 }
@@ -271,7 +359,7 @@ type fakeTestSMTP struct {
 	lock       sync.RWMutex
 }
 
-func (f *fakeTestSMTP) Create(SmtpParams) (smtpClient, error) {
+func (f *fakeTestSMTP) Create(SMTPParams) (smtpClient, error) {
 	if f.fail["create"] {
 		return nil, errors.New("failed to create client")
 	}

backend/app/notify/notify.go

@@ -9,14 +9,15 @@ import (
 
 	log "github.com/go-pkgz/lgr"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // Service delivers notifications to multiple destinations
 type Service struct {
-	dataService  Store
-	destinations []Destination
-	queue        chan Request
+	dataService       Store
+	destinations      []Destination
+	queue             chan Request
+	verificationQueue chan VerificationRequest
 
 	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
 	ctx    context.Context
@@ -26,7 +27,8 @@ type Service struct {
 // Destination defines interface for a given destination service, like telegram, email and so on
 type Destination interface {
 	fmt.Stringer
-	Send(ctx context.Context, req Request) error
+	Send(context.Context, Request) error
+	SendVerification(context.Context, VerificationRequest) error
 }
 
 // Store defines the minimal interface accessing stored comments used by notifier
@@ -35,18 +37,18 @@ type Store interface {
 	GetUserEmail(siteID string, userID string) (string, error)
 }
 
-// Request notification either about comment or about particular user verification
+// Request notification for a Comment
 type Request struct {
-	Comment      store.Comment        // if set sent notifications about new comment
-	parent       store.Comment        // fetched only in case Comment is set
-	Email        string               // if set (also) send email
-	Verification VerificationMetadata // if set sent verification notification
+	Comment     store.Comment
+	parent      store.Comment
+	Emails      []string
 }
 
-// VerificationMetadata required to send notify method verification message
-type VerificationMetadata struct {
+// VerificationRequest notification for user
+type VerificationRequest struct {
 	SiteID string
 	User   string
+	Email  string // if set, send email only
 	Token  string
 }
 
@@ -60,11 +62,12 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	res := Service{
-		dataService:  dataService,
-		queue:        make(chan Request, size),
-		destinations: destinations,
-		ctx:          ctx,
-		cancel:       cancel,
+		dataService:       dataService,
+		queue:             make(chan Request, size),
+		verificationQueue: make(chan VerificationRequest, size),
+		destinations:      destinations,
+		ctx:               ctx,
+		cancel:            cancel,
 	}
 	if len(destinations) > 0 {
 		go res.do()
@@ -78,14 +81,10 @@ func (s *Service) Submit(req Request) {
 	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
 		return
 	}
-	// parent comment is fetched only if comment is present in the Request
 	if s.dataService != nil && req.Comment.ParentID != "" {
 		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
 			req.parent = p
-			req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
-			if err != nil {
-				log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
-			}
+			req.Emails = deduplicateStrings(s.getNotificationEmails(req, p))
 		}
 	}
 	select {
@@ -95,11 +94,45 @@ func (s *Service) Submit(req Request) {
 	}
 }
 
+// getNotificationEmails returns list of emails for notifications for provided comment.
+// Emails is not added to the returned list in case original message is from the same user as the notification receiver.
+func (s *Service) getNotificationEmails(req Request, notifyComment store.Comment) (result []string) {
+	// add current user email only if the user is not the one who wrote the original comment
+	if notifyComment.User.ID != req.Comment.User.ID {
+		email, err := s.dataService.GetUserEmail(req.Comment.Locator.SiteID, notifyComment.User.ID)
+		if err != nil {
+			log.Printf("[WARN] can't read email for %s, %v", notifyComment.User.ID, err)
+		}
+		if email != "" {
+			result = append(result, email)
+		}
+	}
+	if notifyComment.ParentID != "" {
+		if p, err := s.dataService.Get(req.Comment.Locator, notifyComment.ParentID, store.User{}); err == nil {
+			result = append(result, s.getNotificationEmails(req, p)...)
+		}
+	}
+	return result
+}
+
+// SubmitVerification to internal channel if not busy, drop if can't send
+func (s *Service) SubmitVerification(req VerificationRequest) {
+	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
+		return
+	}
+	select {
+	case s.verificationQueue <- req:
+	default:
+		log.Printf("[WARN] can't send verification to queue, %s for %s", req.User, req.Email)
+	}
+}
+
 // Close queue channel and wait for completion
 func (s *Service) Close() {
 	if s.queue != nil {
 		log.Print("[DEBUG] close notifier")
 		close(s.queue)
+		close(s.verificationQueue)
 		s.cancel()
 		<-s.ctx.Done()
 	}
@@ -107,21 +140,60 @@ func (s *Service) Close() {
 }
 
 func (s *Service) do() {
-	for c := range s.queue {
-		var wg sync.WaitGroup
-		wg.Add(len(s.destinations))
-		for _, dest := range s.destinations {
-			go func(d Destination) {
-				if err := d.Send(s.ctx, c); err != nil {
-					log.Printf("[WARN] failed to send to %s, %s", d, err)
-				}
-				wg.Done()
-			}(dest)
+	defer log.Print("[WARN] terminated notifier")
+	var wg sync.WaitGroup
+	for {
+		select {
+		case c, ok := <-s.queue:
+			if !ok {
+				return
+			}
+			wg.Add(len(s.destinations))
+			for _, dest := range s.destinations {
+				go func(d Destination) {
+					if err := d.Send(s.ctx, c); err != nil {
+						log.Printf("[WARN] failed to send to %s, %s", d, err)
+					}
+					wg.Done()
+				}(dest)
+			}
+			wg.Wait()
+		case v, ok := <-s.verificationQueue:
+			if !ok {
+				return
+			}
+			wg.Add(len(s.destinations))
+			for _, dest := range s.destinations {
+				go func(d Destination) {
+					if err := d.SendVerification(s.ctx, v); err != nil {
+						log.Printf("[WARN] failed to send to %s, %s", d, err)
+					}
+					wg.Done()
+				}(dest)
+			}
+			wg.Wait()
+		case <-s.ctx.Done():
+			return
 		}
-		wg.Wait()
 	}
-	log.Print("[WARN] terminated notifier")
 }
 
 // NopService is do-nothing notifier, without destinations
 var NopService = &Service{}
+
+// deduplicateStrings returns provided slice of strings will all duplicates removed.
+// Resulting slice is not sorted.
+func deduplicateStrings(source []string) []string {
+	set := make(map[string]struct{})
+
+	for _, k := range source {
+		set[k] = struct{}{}
+	}
+
+	result := make([]string, 0, len(set))
+	for k := range set {
+		result = append(result, k)
+	}
+
+	return result
+}

backend/app/notify/notify_mock.go

@@ -9,13 +9,16 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// MockDest is a destination mock
 type MockDest struct {
-	data   []Request
-	id     int
-	closed bool
-	lock   sync.Mutex
+	data             []Request
+	verificationData []VerificationRequest
+	id               int
+	closed           bool
+	lock             sync.Mutex
 }
 
+// Send mock
 func (m *MockDest) Send(ctx context.Context, r Request) error {
 	m.lock.Lock()
 	defer m.lock.Unlock()
@@ -30,6 +33,22 @@ func (m *MockDest) Send(ctx context.Context, r Request) error {
 	return nil
 }
 
+// SendVerification mock
+func (m *MockDest) SendVerification(ctx context.Context, v VerificationRequest) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.verificationData = append(m.verificationData, v)
+		log.Printf("sent verification %s -> %d", v.User, m.id)
+	case <-ctx.Done():
+		log.Printf("verification ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+// Get mock
 func (m *MockDest) Get() []Request {
 	m.lock.Lock()
 	defer m.lock.Unlock()
@@ -37,4 +56,14 @@ func (m *MockDest) Get() []Request {
 	copy(res, m.data)
 	return res
 }
+
+// GetVerify mock
+func (m *MockDest) GetVerify() []VerificationRequest {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]VerificationRequest, len(m.verificationData))
+	copy(res, m.verificationData)
+	return res
+}
+
 func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }

backend/app/notify/notify_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestService_NoDestinations(t *testing.T) {
@@ -52,15 +52,42 @@ func TestService_WithDrops(t *testing.T) {
 
 	s.Submit(Request{Comment: store.Comment{ID: "100"}})
 	s.Submit(Request{Comment: store.Comment{ID: "101"}})
-	time.Sleep(time.Millisecond * 11)
 	s.Submit(Request{Comment: store.Comment{ID: "102"}})
-	time.Sleep(time.Millisecond * 11)
+	time.Sleep(time.Millisecond * 21)
 	s.Close()
 
 	s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
 
-	assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
-	assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
+	assert.LessOrEqual(t, len(d1.Get()), 2, "at least one comment from three dropped from d1, got: %v", d1.Get())
+	assert.LessOrEqual(t, len(d2.Get()), 2, "at least one comment from three dropped from d2, got: %v", d2.Get())
+}
+
+func TestService_SubmitVerificationWithDrops(t *testing.T) {
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
+	s := NewService(nil, 1, d1, d2)
+	assert.NotNil(t, s)
+
+	s.SubmitVerification(VerificationRequest{
+		SiteID: "remark",
+		User:   "testUser",
+		Email:  "test@example.org",
+		Token:  "testToken",
+	})
+	s.SubmitVerification(VerificationRequest{})
+	s.SubmitVerification(VerificationRequest{})
+	time.Sleep(time.Millisecond * 21)
+	s.Close()
+
+	s.SubmitVerification(VerificationRequest{}) // safe to send after close
+
+	assert.LessOrEqual(t, len(d2.GetVerify()), 2, "one request from three dropped from d2, got: %v", d2.GetVerify())
+
+	verifyDest := d1.GetVerify()
+	require.LessOrEqual(t, len(verifyDest), 2, "one request from three dropped from d1, got: %v", verifyDest)
+	assert.Equal(t, "remark", verifyDest[0].SiteID)
+	assert.Equal(t, "testUser", verifyDest[0].User)
+	assert.Equal(t, "test@example.org", verifyDest[0].Email)
+	assert.Equal(t, "testToken", verifyDest[0].Token)
 }
 
 func TestService_Many(t *testing.T) {
@@ -70,16 +97,20 @@ func TestService_Many(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
+		s.SubmitVerification(VerificationRequest{User: fmt.Sprintf("%d", 100+i)})
 		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
 	}
 	s.Close()
 	time.Sleep(time.Millisecond * 10)
 
 	assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d1.GetVerify()), "some verifications dropped from d1")
 	assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d2.GetVerify()), "some verifications dropped from d2")
 
 	assert.True(t, d1.closed)
 	assert.True(t, d2.closed)
+	assert.Equal(t, "mock id=1, closed=true", d1.String())
 }
 
 func TestService_WithParent(t *testing.T) {
@@ -106,6 +137,138 @@ func TestService_WithParent(t *testing.T) {
 	assert.Equal(t, "", destRes[1].parent.ID)
 }
 
+func TestService_EmailRetrieval(t *testing.T) {
+	dest := &MockDest{id: 1}
+	dataStore := &mockStore{data: map[string]store.Comment{}, emailData: map[string]string{}}
+
+	dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p1", User: store.User{ID: "u2"}}
+	dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
+	dataStore.emailData["u1"] = "u1@example.com"
+
+	s := NewService(dataStore, 1, dest)
+	assert.NotNil(t, s)
+
+	// one comment, one notification
+	s.Submit(Request{Comment: dataStore.data["p1"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes := dest.Get()
+	require.Equal(t, 1, len(destRes), "one comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ID)
+	assert.Empty(t, destRes[0].parent)
+	assert.Empty(t, destRes[0].Emails)
+
+	// reply to the first comment, same comment as one in original comment
+	s.Submit(Request{Comment: dataStore.data["p2"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p2", destRes[1].Comment.ID)
+	assert.Equal(t, "p1", destRes[1].parent.ID)
+	assert.Equal(t, "u1", destRes[1].parent.User.ID)
+	assert.Empty(t, destRes[1].Emails, "u1 is not notified they are the one who left the comment")
+
+	// another reply to the first comment, another user
+	s.Submit(Request{Comment: dataStore.data["p3"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 3, len(destRes), "three comment notified")
+	assert.Equal(t, "p3", destRes[2].Comment.ID)
+	assert.Equal(t, "p1", destRes[2].parent.ID)
+	assert.Equal(t, "u1", destRes[2].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
+
+	// reply to the last comment by another user, should trigger email retrieval error
+	s.Submit(Request{Comment: dataStore.data["p4"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 4, len(destRes), "four comment notified")
+	assert.Equal(t, "p4", destRes[3].Comment.ID)
+	assert.Equal(t, "p3", destRes[3].parent.ID)
+	assert.Equal(t, "u2", destRes[3].parent.User.ID)
+	assert.Empty(t, destRes[3].Emails, "no email can be retrieved for u2")
+
+	s.Close()
+}
+
+func TestService_Recursive(t *testing.T) {
+	dest := &MockDest{id: 1}
+	dataStore := &mockStore{data: map[string]store.Comment{}, emailData: map[string]string{}}
+
+	dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u2"}}
+	dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p2", User: store.User{ID: "u3"}}
+	dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
+	dataStore.data["p5"] = store.Comment{ID: "p5", ParentID: "p4", User: store.User{ID: "u4"}}
+	dataStore.emailData["u1"] = "u1@example.com"
+	// second comment goes without email address for notification
+	dataStore.emailData["u3"] = "u3@example.com"
+
+	s := NewService(dataStore, 1, dest)
+	assert.NotNil(t, s)
+
+	// one comment from u1 with email set
+	s.Submit(Request{Comment: dataStore.data["p1"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes := dest.Get()
+	require.Equal(t, 1, len(destRes), "one comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ID)
+	assert.Empty(t, destRes[0].parent)
+	assert.Empty(t, destRes[0].Emails)
+
+	// reply to the first comment from u2 without email set
+	s.Submit(Request{Comment: dataStore.data["p2"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p2", destRes[1].Comment.ID)
+	assert.Equal(t, "p1", destRes[1].parent.ID)
+	assert.Equal(t, "u1", destRes[1].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[1].Emails)
+
+	// reply to the second comment from u3 with email set
+	s.Submit(Request{Comment: dataStore.data["p3"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 3, len(destRes), "three comment notified")
+	assert.Equal(t, "p3", destRes[2].Comment.ID)
+	assert.Equal(t, "p2", destRes[2].parent.ID)
+	assert.Equal(t, "u2", destRes[2].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
+
+	// reply to the third comment from u1 (author of the first comment), only u3 should be notified
+	s.Submit(Request{Comment: dataStore.data["p4"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 4, len(destRes), "four comment notified once each")
+	assert.Equal(t, "p4", destRes[3].Comment.ID)
+	assert.Equal(t, "p3", destRes[3].parent.ID)
+	assert.Equal(t, "u3", destRes[3].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u3@example.com"}, destRes[3].Emails, "u1 is not notified they are the one who left the comment")
+
+	// reply to the fourth comment from u4, u1 and u3 should be notified once as a result
+	s.Submit(Request{Comment: dataStore.data["p5"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 5, len(destRes), "four comment notified once each")
+	assert.Equal(t, "p5", destRes[4].Comment.ID)
+	assert.Equal(t, "p4", destRes[4].parent.ID)
+	assert.Equal(t, "u1", destRes[4].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com", "u3@example.com"}, destRes[4].Emails, "u3 and u1 notified once")
+
+	s.Close()
+}
+
 func TestService_Nop(t *testing.T) {
 	s := NopService
 	s.Submit(Request{Comment: store.Comment{}})
@@ -113,7 +276,10 @@ func TestService_Nop(t *testing.T) {
 	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
 }
 
-type mockStore struct{ data map[string]store.Comment }
+type mockStore struct {
+	data      map[string]store.Comment
+	emailData map[string]string
+}
 
 func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
 	res, ok := m.data[id]
@@ -123,6 +289,10 @@ func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment,
 	return res, nil
 }
 
-func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
-	return "", errors.New("no such user")
+func (m mockStore) GetUserEmail(_, userID string) (string, error) {
+	email, ok := m.emailData[userID]
+	if !ok {
+		return "", errors.New("no such user")
+	}
+	return email, nil
 }

backend/app/notify/telegram.go

@@ -8,6 +8,7 @@ import (
 	"html"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
@@ -27,8 +28,7 @@ const telegramTimeOut = 5000 * time.Millisecond
 const telegramAPIPrefix = "https://api.telegram.org/bot"
 
 // NewTelegram makes telegram bot for notifications
-func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
-
+func NewTelegram(token, channelID string, timeout time.Duration, api string) (*Telegram, error) {
 	if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
 		channelID = "@" + channelID // if channelID not a number enforce @ prefix
 	}
@@ -86,10 +86,6 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 
 // Send to telegram channel
 func (t *Telegram) Send(ctx context.Context, req Request) error {
-	if req.Comment.ID == "" {
-		// verification request received, send nothing
-		return nil
-	}
 	client := http.Client{Timeout: telegramTimeOut}
 	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
 
@@ -100,7 +96,7 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
 	from = "*" + from + "*"
 	link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
 	if req.Comment.PostTitle != "" {
-		link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
+		link = fmt.Sprintf("↦ [%s](%s)", t.escapeTitle(req.Comment.PostTitle), req.Comment.Locator.URL+uiNav+req.Comment.ID)
 	}
 	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
 		t.apiPrefix, t.token, t.channelID)
@@ -147,6 +143,20 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
 	return nil
 }
 
+func (t *Telegram) escapeTitle(title string) string {
+	escSymbols := []string{"[", "]", "(", ")"}
+	res := title
+	for _, esc := range escSymbols {
+		res = strings.Replace(res, esc, "\\"+esc, -1)
+	}
+	return res
+}
+
+// SendVerification is not implemented for telegram
+func (t *Telegram) SendVerification(_ context.Context, _ VerificationRequest) error {
+	return nil
+}
+
 func (t *Telegram) String() string {
 	return "telegram: " + t.channelID
 }

backend/app/notify/telegram_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 	"time"
 
@@ -11,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestTelegram_New(t *testing.T) {
@@ -37,7 +38,9 @@ func TestTelegram_New(t *testing.T) {
 	assert.EqualError(t, err, "unexpected telegram status code 404")
 
 	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
-	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "can't initialize telegram notifications")
+	assert.Contains(t, err.Error(), "dial tcp 127.0.0.1:4321: connect: connection refused")
 
 	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
 	assert.Error(t, err, "empty api url not allowed")
@@ -69,6 +72,12 @@ func TestTelegram_Send(t *testing.T) {
 	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
+	assert.NoError(t, err)
+	c.PostTitle = "[test title]"
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
+	assert.NoError(t, err)
+
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.Error(t, err, "should failed")
 	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
@@ -76,7 +85,18 @@ func TestTelegram_Send(t *testing.T) {
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())
-	require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
+}
+
+func TestTelegram_SendVerification(t *testing.T) {
+	ts := mockTelegramServer()
+	defer ts.Close()
+
+	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NoError(t, err)
+	assert.NotNil(t, tb)
+
+	err = tb.SendVerification(context.TODO(), VerificationRequest{})
+	assert.NoError(t, err)
 }
 
 func mockTelegramServer() *httptest.Server {
@@ -121,3 +141,26 @@ func mockTelegramServer() *httptest.Server {
 
 	return httptest.NewServer(router)
 }
+
+func TestTelegram_escapeTitle(t *testing.T) {
+
+	tbl := []struct {
+		inp string
+		out string
+	}{
+		{"", ""},
+		{"something 123", "something 123"},
+		{"something [123]", "something \\[123\\]"},
+		{"something (123)", "something \\(123\\)"},
+		{"something (123) [aaa]", "something \\(123\\) \\[aaa\\]"},
+	}
+
+	tb := Telegram{}
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.out, tb.escapeTitle(tt.inp))
+		})
+	}
+
+}

backend/app/notify/testdata/bad.html.tmpl

@@ -0,0 +1 @@
+{{

backend/app/notify/testdata/msg.html.tmpl

@@ -0,0 +1,20 @@
+{{- if .ForAdmin}}
+New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
+{{- else }}
+	New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
+{{- end }}
+{{- if .ParentCommentText}}
+	{{.ParentUserPicture}}
+	{{.ParentUserName}}
+	{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}
+	Parent comment link: {{.ParentCommentLink}}
+	{{.ParentCommentText}}
+{{- end }}
+
+User: {{.UserName}}
+{{.CommentDate.Format "02.01.2006 at 15:04"}}
+Comment: {{.CommentText}}
+{{.Email}} {{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}
+{{- if .UnsubscribeLink}}
+Unsubscribe link: {{.UnsubscribeLink}}
+{{- end }}

backend/app/notify/testdata/verification.html.tmpl

@@ -0,0 +1,7 @@
+Confirmation for {{.User}} on site {{.Site}}
+{{- if .SubscribeURL}}
+Subscribe url: {{.SubscribeURL}}{{.Token}}
+{{- end }}
+Token:{{.Token}}
+Sent to {{.Email}}
+

backend/app/rest/api/admin.go

@@ -13,9 +13,9 @@ import (
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
 )
 
 // admin provides router for all requests available for admin users only
@@ -111,7 +111,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
+	if err = a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
 		code := parseError(err, rest.ErrInternal)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
 		return

backend/app/rest/api/admin_test.go

@@ -13,16 +13,15 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestAdmin_Delete(t *testing.T) {
@@ -48,24 +47,24 @@ func TestAdmin_Delete(t *testing.T) {
 	// check multi count
 	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, resp.Body.Close())
 	assert.NoError(t, err)
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 
 	// delete a comment
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	require.NoError(t, err)
-	defer resp.Body.Close()
 	requireAdminOnly(t, req)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
@@ -99,12 +98,13 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err = ioutil.ReadAll(resp.Body)
+	assert.NoError(t, resp.Body.Close())
 	assert.NoError(t, err)
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 }
 
 func TestAdmin_Title(t *testing.T) {
@@ -141,6 +141,7 @@ func TestAdmin_Title(t *testing.T) {
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=%s/post1", ts.URL, id1, tss.URL))
@@ -175,6 +176,7 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
@@ -224,6 +226,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
 		return resp.StatusCode
 	}
 
@@ -310,7 +313,7 @@ func TestAdmin_Block(t *testing.T) {
 	pi = []store.PostInfo{}
 	err = json.Unmarshal(body, &pi)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}), pi)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}, pi)
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
@@ -380,6 +383,7 @@ func TestAdmin_BlockedList(t *testing.T) {
 	assert.NoError(t, err)
 	res, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, res.Body.Close())
 	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
@@ -388,6 +392,7 @@ func TestAdmin_BlockedList(t *testing.T) {
 	assert.NoError(t, err)
 	res, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, res.Body.Close())
 	assert.Equal(t, 200, res.StatusCode)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", nil)
@@ -398,6 +403,7 @@ func TestAdmin_BlockedList(t *testing.T) {
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.NoError(t, err)
+	require.NoError(t, res.Body.Close())
 	require.Equal(t, 2, len(users), "two users blocked")
 	assert.Equal(t, "user1", users[0].ID)
 	assert.Equal(t, "user1 name", users[0].Name)
@@ -414,6 +420,7 @@ func TestAdmin_BlockedList(t *testing.T) {
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.NoError(t, err)
+	require.NoError(t, res.Body.Close())
 	assert.Equal(t, 1, len(users), "one user left blocked")
 }
 
@@ -441,9 +448,11 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err := sendReq(t, req, "") // non-admin user
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 401, resp.StatusCode)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
@@ -458,6 +467,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	require.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// reset post's read-only
@@ -466,6 +476,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
@@ -480,6 +491,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	require.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
@@ -494,6 +506,7 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.Error(t, err)
@@ -529,6 +542,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
@@ -540,6 +554,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 403, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
@@ -569,6 +584,7 @@ func TestAdmin_Verify(t *testing.T) {
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("remark42", "user1")
 	assert.True(t, verified)
@@ -587,6 +603,7 @@ func TestAdmin_Verify(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("remark42", "user1")
 	assert.False(t, verified)
@@ -707,6 +724,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
 
 	_, err = srv.DataService.User("remark42", "user1", 0, 0, store.User{})
@@ -739,6 +757,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, 400, resp.StatusCode)
 
 	// try with bad auth
@@ -766,6 +785,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	req.SetBasicAuth("admin", "bad-password")
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, 403, resp.StatusCode)
 
 	// try bad user
@@ -778,6 +798,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, 400, resp.StatusCode, resp.Status)
 
 	// try without deleteme flag
@@ -793,6 +814,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	assert.Equal(t, 403, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.True(t, strings.Contains(string(b), "can't use provided token"))
 }
 

backend/app/rest/api/migrator.go

@@ -17,8 +17,8 @@ import (
 	R "github.com/go-pkgz/rest"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/migrator"
+	"github.com/umputun/remark42/backend/app/rest"
 )
 
 // Migrator rest with import and export controllers
@@ -28,7 +28,7 @@ type Migrator struct {
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
 	NativeExporter    migrator.Exporter
-	UrlMapperMaker    migrator.MapperMaker
+	URLMapperMaker    migrator.MapperMaker
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -37,7 +37,7 @@ type Migrator struct {
 
 // KeyStore defines sub-interface for consumers needed just a key
 type KeyStore interface {
-	Key() (key string, err error)
+	Key(siteID string) (key string, err error)
 }
 
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
@@ -161,7 +161,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	// create new url-mapper from given rules in body
-	mapper, err := m.UrlMapperMaker(r.Body)
+	mapper, err := m.URLMapperMaker(r.Body)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
 		return
@@ -180,12 +180,12 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		defer func() {
-			if e := os.Remove(fh.Name()); e != nil {
+			if e = os.Remove(fh.Name()); e != nil {
 				log.Printf("[WARN] failed to remove temp file %+v", e)
 			}
 		}()
 		log.Printf("[DEBUG] start export for site=%s", siteID)
-		if _, e := m.NativeExporter.Export(fh, siteID); e != nil {
+		if _, e = m.NativeExporter.Export(fh, siteID); e != nil {
 			log.Printf("[WARN] export failed with %+v", e)
 			return
 		}
@@ -212,7 +212,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // runImport reads from tmpfile and import for given siteID and provider
-func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
+func (m *Migrator) runImport(siteID, provider, tmpfile string) {
 	m.setBusy(siteID, true)
 
 	defer func() {
@@ -233,7 +233,7 @@ func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	}
 	log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, provider)
 
-	fh, err := os.Open(tmpfile)
+	fh, err := os.Open(tmpfile) // nolint
 	if err != nil {
 		log.Printf("[WARN] import failed, %v", err)
 		return

backend/app/rest/api/migrator_test.go

@@ -17,8 +17,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
@@ -124,6 +124,7 @@ func TestMigrator_ImportRejected(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err := client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
@@ -147,6 +148,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err := client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 5 * time.Second}
@@ -156,6 +158,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 }
@@ -168,11 +171,12 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 "picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
 "ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
 "votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
-	recs := []string{}
-	for i := 0; i < 50; i++ {
+	nRecs := 50
+	recs := make([]string, 0, nRecs)
+	for i := 0; i < nRecs; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
-	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with `nRecs` records
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
@@ -180,15 +184,17 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	require.NoError(t, err)
 	resp, err := client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 5 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=10ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=5ms", nil)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
 	waitForMigrationCompletion(t, ts)
@@ -214,6 +220,7 @@ func TestMigrator_Export(t *testing.T) {
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 
@@ -230,6 +237,7 @@ func TestMigrator_Export(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
 	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
 	t.Logf("%s", string(ungzBody))
@@ -245,6 +253,7 @@ func TestMigrator_Export(t *testing.T) {
 
 	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 3, strings.Count(string(body), "\n"))
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
@@ -253,6 +262,7 @@ func TestMigrator_Export(t *testing.T) {
 	require.NoError(t, err)
 	resp, err = client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
@@ -304,6 +314,7 @@ func TestMigrator_Remap(t *testing.T) {
 	rules := "https://remark42.com/* https://www.remark42.com/*"
 	resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 
@@ -351,6 +362,7 @@ func TestMigrator_RemapReject(t *testing.T) {
 	require.NoError(t, err)
 	resp, err := client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
@@ -457,7 +469,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/rest/api/rest.go

@@ -6,12 +6,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/didip/tollbooth"
+	"github.com/didip/tollbooth/v6"
 	"github.com/didip/tollbooth_chi"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
@@ -25,12 +24,13 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rakyll/statik/fs"
 
-	"github.com/umputun/remark/backend/app/notify"
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/rest/proxy"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/image"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/templates"
 )
 
 // Rest is a rest access server
@@ -45,7 +45,6 @@ type Rest struct {
 	Migrator         *Migrator
 	NotifyService    *notify.Service
 	ImageService     *image.Service
-	Streamer         *Streamer
 
 	AnonVote        bool
 	WebRoot         string
@@ -60,6 +59,9 @@ type Rest struct {
 	EmailNotifications bool
 	EmojiEnabled       bool
 	SimpleView         bool
+	ProxyCORS          bool
+	SendJWTHeader      bool
+	AllowedAncestors   []string // sets Content-Security-Policy "frame-ancestors ..."
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
@@ -76,6 +78,7 @@ type Rest struct {
 type LoadingCache interface {
 	Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
 	Flush(req lcw.FlusherRequest)                                        // evict matched records
+	Close() error
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
@@ -184,15 +187,24 @@ func (s *Rest) routes() chi.Router {
 
 	s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
 
-	corsMiddleware := cors.New(cors.Options{
-		AllowedOrigins:   []string{"*"},
-		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
-		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
-		ExposedHeaders:   []string{"Authorization"},
-		AllowCredentials: true,
-		MaxAge:           300,
-	})
-	router.Use(corsMiddleware.Handler)
+	if s.ProxyCORS {
+		log.Printf("[WARN] internal CORS disabled")
+	} else {
+		corsMiddleware := cors.New(cors.Options{
+			AllowedOrigins:   []string{"*"},
+			AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
+			AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
+			ExposedHeaders:   []string{"Authorization"},
+			AllowCredentials: true,
+			MaxAge:           300,
+		})
+		router.Use(corsMiddleware.Handler)
+	}
+
+	if len(s.AllowedAncestors) > 0 {
+		log.Printf("[INFO] allowed from %+v only", s.AllowedAncestors)
+		router.Use(frameAncestors(s.AllowedAncestors))
+	}
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
 	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
@@ -201,13 +213,13 @@ func (s *Rest) routes() chi.Router {
 
 	router.Group(func(r chi.Router) {
 		r.Use(middleware.Timeout(5 * time.Second))
-		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
 		r.Use(middleware.Timeout(5 * time.Second))
-		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
+		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
 		r.Mount("/avatar", avatarHandler)
 	})
 
@@ -219,7 +231,6 @@ func (s *Rest) routes() chi.Router {
 		rapi.Group(func(rava chi.Router) {
 			rava.Use(middleware.Timeout(5 * time.Second))
 			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
-			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
 		})
 
@@ -248,14 +259,6 @@ func (s *Rest) routes() chi.Router {
 
 		})
 
-		// open routes, streams, no send timeout
-		rapi.Route("/stream", func(rstream chi.Router) {
-			rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
-			rstream.Get("/info", s.pubRest.infoStreamCtrl)
-			rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
-		})
-
 		// open routes, cached
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(middleware.Timeout(30 * time.Second))
@@ -304,8 +307,7 @@ func (s *Rest) routes() chi.Router {
 			rauth.Use(middleware.Timeout(10 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
 			rauth.Use(authMiddleware.Auth, matchSiteID)
-			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(middleware.NoCache, logInfoWithBody)
 
 			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
 			rauth.Post("/comment", s.privRest.createCommentCtrl)
@@ -352,7 +354,6 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 		commentFormatter: s.CommentFormatter,
 		readOnlyAge:      s.ReadOnlyAge,
 		webRoot:          s.WebRoot,
-		streamer:         s.Streamer,
 	}
 
 	privGrp := private{
@@ -365,6 +366,7 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 		notifyService:    s.NotifyService,
 		remarkURL:        s.RemarkURL,
 		anonVote:         s.AnonVote,
+		templates:        templates.NewFS(),
 	}
 
 	admGrp := admin{
@@ -415,6 +417,7 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		EmailNotifications bool     `json:"email_notifications"`
 		EmojiEnabled       bool     `json:"emoji_enabled"`
 		SimpleView         bool     `json:"simple_view"`
+		SendJWTHeader      bool     `json:"send_jwt_header"`
 	}{
 		Version:            s.Version,
 		EditDuration:       int(s.DataService.EditDuration.Seconds()),
@@ -425,11 +428,12 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		CriticalScore:      s.ScoreThresholds.Critical,
 		PositiveScore:      s.DataService.PositiveScore,
 		ReadOnlyAge:        s.ReadOnlyAge,
-		MaxImageSize:       s.ImageService.Store.SizeLimit(),
+		MaxImageSize:       s.ImageService.MaxSize,
 		EmailNotifications: s.EmailNotifications,
 		EmojiEnabled:       s.EmojiEnabled,
 		AnonVote:           s.AnonVote,
 		SimpleView:         s.SimpleView,
+		SendJWTHeader:      s.SendJWTHeader,
 	}
 
 	cnf.Auth = []string{}
@@ -462,7 +466,7 @@ func addFileServer(r chi.Router, path string, root http.FileSystem, version stri
 	origPath := path
 	webFS = http.StripPrefix(path, webFS)
 	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
+		r.Get(path, http.RedirectHandler(path+"/", http.StatusMovedPermanently).ServeHTTP)
 		path += "/"
 	}
 	path += "*"
@@ -581,7 +585,7 @@ func cacheControl(expiration time.Duration, version string) func(http.Handler) h
 		fn := func(w http.ResponseWriter, r *http.Request) {
 			e := `"` + etag(r, version) + `"`
 			w.Header().Set("Etag", e)
-			w.Header().Set("Cache-Control", "max-age="+strconv.Itoa(int(expiration.Seconds())))
+			w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, no-cache", int(expiration.Seconds())))
 
 			if match := r.Header.Get("If-None-Match"); match != "" {
 				if strings.Contains(match, e) {
@@ -595,6 +599,22 @@ func cacheControl(expiration time.Duration, version string) func(http.Handler) h
 	}
 }
 
+// frameAncestors is a middleware setting Content-Security-Policy "frame-ancestors host1 host2 ..."
+// prevents loading of comments widgets from any other origins. In case if the list of allowed empty, ignored.
+func frameAncestors(hosts []string) func(http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+			if len(hosts) == 0 {
+				h.ServeHTTP(w, r)
+				return
+			}
+			w.Header().Set("Content-Security-Policy", "frame-ancestors "+strings.Join(hosts, " ")+";")
+			h.ServeHTTP(w, r)
+		}
+		return http.HandlerFunc(fn)
+	}
+}
+
 func parseError(err error, defaultCode int) (code int) {
 	code = defaultCode
 

backend/app/rest/api/rest_private.go

@@ -22,12 +22,13 @@ import (
 	R "github.com/go-pkgz/rest"
 	"github.com/hashicorp/go-multierror"
 
-	"github.com/umputun/remark/backend/app/notify"
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/image"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/templates"
 )
 
 type private struct {
@@ -40,6 +41,7 @@ type private struct {
 	authenticator    *auth.Service
 	remarkURL        string
 	anonVote         bool
+	templates        templates.FileReader
 }
 
 type privStore interface {
@@ -58,21 +60,6 @@ type privStore interface {
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 }
 
-const unsubscribeHtml = `<!DOCTYPE html>
-<html>
-<head>
-    <meta name="viewport" content="width=device-width"/>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-</head>
-<body>
-<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
-    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
-	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Successfully unsubscribed</p>
-</div>
-</body>
-</html>
-`
-
 // POST /comment - adds comment, resets all immutable fields
 func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -276,7 +263,8 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
 	address := r.URL.Query().Get("address")
 	siteID := r.URL.Query().Get("site")
 	if address == "" {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest,
+			errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
 		return
 	}
 	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
@@ -284,7 +272,8 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
 		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
 	}
 	if address == existingAddress {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusConflict,
+			errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
 		return
 	}
 	claims := token.Claims{
@@ -303,14 +292,12 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	s.notifyService.Submit(
-		notify.Request{
-			Email: address,
-			Verification: notify.VerificationMetadata{
-				SiteID: siteID,
-				User:   user.Name,
-				Token:  tkn,
-			},
+	s.notifyService.SubmitVerification(
+		notify.VerificationRequest{
+			SiteID: siteID,
+			User:   user.Name,
+			Email:  address,
+			Token:  tkn,
 		},
 	)
 
@@ -373,25 +360,28 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request)
 func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 	tkn := r.URL.Query().Get("tkn")
 	if tkn == "" {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest,
+			errors.New("missing parameter"), "token parameter is required", rest.ErrInternal, s.templates)
 		return
 	}
 	siteID := r.URL.Query().Get("site")
 
 	confClaims, err := s.authenticator.TokenService().Parse(tkn)
 	if err != nil {
-		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal, s.templates)
 		return
 	}
 
 	if s.authenticator.TokenService().IsExpired(confClaims) {
-		rest.SendErrorHTML(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusForbidden,
+			errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal, s.templates)
 		return
 	}
 
 	elems := strings.Split(confClaims.Handshake.ID, "::")
 	if len(elems) != 2 {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest,
+			errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal, s.templates)
 		return
 	}
 	userID := elems[0]
@@ -402,19 +392,22 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't read email for %s, %v", userID, err)
 	}
 	if existingAddress == "" {
-		rest.SendErrorHTML(w, r, http.StatusConflict, errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusConflict,
+			errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal, s.templates)
 		return
 	}
 	if address != existingAddress {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest,
+			errors.New("wrong email unsubscription"), "email address in request does not match known for this user",
+			rest.ErrInternal, s.templates)
 		return
 	}
 
 	log.Printf("[DEBUG] unsubscribe user %s", userID)
 
-	if err := s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
+	if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
 		code := parseError(err, rest.ErrInternal)
-		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code, s.templates)
 		return
 	}
 	// clean User.Email from the token, if user has the token
@@ -425,7 +418,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 	if claims.User != nil && claims.User.Email != "" {
 		claims.User.Email = ""
 		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
-			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal, s.templates)
 			return
 		}
 	}
@@ -436,8 +429,15 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 			panic(err)
 		}
 	}
-
-	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHtml))
+	MustRead := func(path string) string {
+		file, err := s.templates.ReadFile(path)
+		if err != nil {
+			panic(err)
+		}
+		return string(file)
+	}
+	tmplstr := MustRead("unsubscribe.html.tmpl")
+	tmpl := template.Must(template.New("unsubscribe").Parse(tmplstr))
 	msg := bytes.Buffer{}
 	MustExecute(tmpl, &msg, nil)
 	render.HTML(w, r, msg.String())
@@ -568,14 +568,14 @@ func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	file, header, err := r.FormFile("file")
+	file, _, err := r.FormFile("file")
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
-	id, err := s.imageService.Save(header.Filename, user.ID, file)
+	id, err := s.imageService.Save(user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return

backend/app/rest/api/rest_private_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -23,9 +24,9 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/notify"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/image"
 )
 
 // gopher png for test, from https://golang.org/src/image/png/example_test.go
@@ -71,6 +72,7 @@ func TestRest_CreateOldPost(t *testing.T) {
 	resp, err := post(t, ts.URL+"/api/v1/comment",
 		`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 
 	assert.NoError(t, srv.DataService.DeleteAll("remark42"))
@@ -83,6 +85,7 @@ func TestRest_CreateOldPost(t *testing.T) {
 	resp, err = post(t, ts.URL+"/api/v1/comment",
 		`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 }
 
@@ -120,8 +123,8 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1", 
-"site": "remark42"}}`)
+	badComment := `{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1",
+"site": "remark42"}}`
 
 	resp, err := post(t, ts.URL+"/api/v1/comment", badComment)
 	assert.NoError(t, err)
@@ -144,6 +147,7 @@ func TestRest_CreateRejected(t *testing.T) {
 	// try to create without auth
 	resp, err := http.Post(ts.URL+"/api/v1/comment", "", strings.NewReader(body))
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 401, resp.StatusCode)
 
 	// try with wrong aud
@@ -153,9 +157,26 @@ func TestRest_CreateRejected(t *testing.T) {
 	req.Header.Add("X-JWT", devTokenBadAud)
 	resp, err = client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusForbidden, resp.StatusCode, "reject wrong aud")
 }
 
+func TestRest_CreateWithLazyImage(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+	body := `{"text": "test 123 ![](http://example.com/image.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
+	// create comment
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	c := store.Comment{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	assert.Equal(t, c.Text, "<p>test 123 <img src=\"http://example.com/image.png\" alt=\"\" loading=\"lazy\"/></p>\n")
+}
+
 func TestRest_CreateAndGet(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -249,8 +270,8 @@ func TestRest_UpdateDelete(t *testing.T) {
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 
 	// delete a comment
 	client := http.Client{}
@@ -290,8 +311,8 @@ func TestRest_UpdateDelete(t *testing.T) {
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	require.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
-		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 }
 
 func TestRest_UpdateNotOwner(t *testing.T) {
@@ -312,6 +333,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	assert.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
 	assert.NoError(t, err)
+	assert.NoError(t, b.Body.Close())
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
 	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
@@ -322,6 +344,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	req.Header.Add("X-JWT", devToken)
 	b, err = client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, b.Body.Close())
 	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
 }
 
@@ -340,6 +363,7 @@ func TestRest_UpdateWrongAud(t *testing.T) {
 	req.Header.Add("X-JWT", devTokenBadAud)
 	b, err := client.Do(req)
 	assert.NoError(t, err)
+	assert.NoError(t, b.Body.Close())
 	assert.Equal(t, http.StatusForbidden, b.StatusCode, "reject update with wrong aut in jwt")
 }
 
@@ -388,6 +412,7 @@ func TestRest_Vote(t *testing.T) {
 		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
 		return resp.StatusCode
 	}
 
@@ -400,7 +425,8 @@ func TestRest_Vote(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 1, cr.Score)
 	assert.Equal(t, 1, cr.Vote)
-	assert.Equal(t, map[string]bool(nil), cr.Votes)
+	assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
+	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
 	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
@@ -436,7 +462,8 @@ func TestRest_Vote(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
-	assert.Equal(t, map[string]bool(nil), cr.Votes)
+	assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
+	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 
 	req, err := http.NewRequest("GET",
 		fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
@@ -449,7 +476,8 @@ func TestRest_Vote(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
-	assert.Equal(t, map[string]bool(nil), cr.Votes)
+	assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
+	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 }
 
 func TestRest_AnonVote(t *testing.T) {
@@ -472,6 +500,7 @@ func TestRest_AnonVote(t *testing.T) {
 		req.Header.Add("X-JWT", anonToken)
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
 		return resp.StatusCode
 	}
 
@@ -501,12 +530,21 @@ func TestRest_AnonVote(t *testing.T) {
 	assert.Equal(t, 1, cr.Score)
 	assert.Equal(t, 1, cr.Vote)
 	assert.Equal(t, map[string]bool(nil), cr.Votes)
+	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs)
+}
+
+type MockFS struct{}
+
+func (fs *MockFS) ReadFile(path string) ([]byte, error) {
+	return []byte(fmt.Sprintf("template %s", path)), nil
 }
 
 func TestRest_Email(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
+	srv.privRest.templates = &MockFS{}
+
 	// issue good token
 	claims := token.Claims{
 		Handshake: &token.Handshake{ID: "dev::good@example.com"},
@@ -548,6 +586,7 @@ func TestRest_Email(t *testing.T) {
 	}
 	client := http.Client{}
 	for _, x := range testData {
+		x := x
 		t.Run(x.description, func(t *testing.T) {
 			req, err := http.NewRequest(x.method, ts.URL+x.url, nil)
 			require.NoError(t, err)
@@ -577,6 +616,7 @@ func TestRest_EmailNotification(t *testing.T) {
 
 	mockDestination := &notify.MockDest{}
 	srv.privRest.notifyService = notify.NewService(srv.DataService, 1, mockDestination)
+	defer srv.privRest.notifyService.Close()
 
 	client := http.Client{}
 
@@ -592,15 +632,16 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.NoError(t, err)
 	body, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	parentComment := store.Comment{}
 	require.NoError(t, render.DecodeJSON(strings.NewReader(string(body)), &parentComment))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
 	require.Equal(t, 1, len(mockDestination.Get()))
-	assert.Equal(t, "", mockDestination.Get()[0].Email)
+	assert.Empty(t, mockDestination.Get()[0].Emails)
 
-	// create child comment from another user, no email notification expected
+	// create child comment from another user, email notification only to admin expected
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
 		`{"text": "test 456",
 	"pid": "%s",
@@ -608,16 +649,17 @@ func TestRest_EmailNotification(t *testing.T) {
 	"locator":{"url": "https://radio-t.com/blah1",
 	"site": "remark42"}}`, parentComment.ID)))
 	assert.NoError(t, err)
-	req.Header.Add("X-JWT", devToken)
+	req.Header.Add("X-JWT", anonToken)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
+	time.Sleep(time.Millisecond * 30)
 	require.Equal(t, 2, len(mockDestination.Get()))
-	assert.Empty(t, mockDestination.Get()[1].Email)
+	assert.Empty(t, mockDestination.Get()[1].Emails)
 
 	// send confirmation token for email
 	req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", nil)
@@ -627,12 +669,13 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
-	require.Equal(t, 3, len(mockDestination.Get()))
-	require.NotEmpty(t, mockDestination.Get()[2].Verification)
-	verificationToken := mockDestination.Get()[2].Verification.Token
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 1, len(mockDestination.GetVerify()))
+	assert.Equal(t, "good@example.com", mockDestination.GetVerify()[0].Email)
+	verificationToken := mockDestination.GetVerify()[0].Token
 
 	// verify email
 	req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), nil)
@@ -642,6 +685,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 
 	// get user information to verify the subscription
@@ -652,6 +696,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 	var user store.User
 	err = json.Unmarshal(body, &user)
@@ -667,16 +712,17 @@ func TestRest_EmailNotification(t *testing.T) {
 	"locator":{"url": "https://radio-t.com/blah1",
 	"site": "remark42"}}`, parentComment.ID)))
 	assert.NoError(t, err)
-	req.Header.Add("X-JWT", devToken)
+	req.Header.Add("X-JWT", anonToken)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
-	require.Equal(t, 4, len(mockDestination.Get()))
-	assert.Equal(t, "good@example.com", mockDestination.Get()[3].Email)
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 3, len(mockDestination.Get()))
+	assert.Equal(t, []string{"good@example.com"}, mockDestination.Get()[2].Emails)
 
 	// delete user's email
 	req, err = http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/email?site=remark42", nil)
@@ -686,9 +732,10 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 
-	// create child comment from another user, no email notification expected
+	// create child comment from another user, no email notification
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 321",
 	"user": {"name": "other_user"},
@@ -700,11 +747,12 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.NoError(t, err)
 	body, err = ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
 	// wait for mock notification Submit to kick off
-	time.Sleep(time.Millisecond * 5)
-	require.Equal(t, 5, len(mockDestination.Get()))
-	assert.Empty(t, mockDestination.Get()[4].Email)
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 4, len(mockDestination.Get()))
+	assert.Empty(t, mockDestination.Get()[3].Emails)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -714,11 +762,11 @@ func TestRest_UserAllData(t *testing.T) {
 	// write 3 comments
 	user := store.User{ID: "dev", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
 	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
 	_, err := srv.DataService.Create(c1)
 	require.NoError(t, err, "%+v", err)
 	_, err = srv.DataService.Create(c2)
@@ -737,6 +785,7 @@ func TestRest_UserAllData(t *testing.T) {
 
 	ungzReader, err := gzip.NewReader(resp.Body)
 	assert.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
 	strUungzBody := string(ungzBody)
@@ -759,6 +808,7 @@ func TestRest_UserAllData(t *testing.T) {
 	require.NoError(t, err)
 	resp, err = client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	require.Equal(t, 401, resp.StatusCode)
 }
 
@@ -768,7 +818,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 
 	user := store.User{ID: "dev", Name: "user name 1"}
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 
 	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
@@ -807,6 +857,7 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, resp.Body.Close())
 	assert.NoError(t, err)
 
 	m := map[string]string{}
@@ -815,17 +866,18 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Equal(t, "remark42", m["site"])
 	assert.Equal(t, "dev", m["user_id"])
 
-	token := m["token"]
-	claims, err := srv.Authenticator.TokenService().Parse(token)
+	tkn := m["token"]
+	claims, err := srv.Authenticator.TokenService().Parse(tkn)
 	assert.NoError(t, err)
 	assert.Equal(t, "dev", claims.User.ID)
-	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"])
+	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+tkn, m["link"])
 
 	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), nil)
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
 }
 
 func TestRest_SavePictureCtrl(t *testing.T) {
@@ -853,6 +905,7 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 		assert.Equal(t, 200, resp.StatusCode)
 		body, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
+		require.NoError(t, resp.Body.Close())
 
 		m := map[string]string{}
 		err = json.Unmarshal(body, &m)
@@ -867,29 +920,34 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 1462, len(body))
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.gif")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.jpg")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.blah")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
@@ -901,13 +959,14 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	}()
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
-	imageService := svc.ImageService
-	imageService.Store = &image.FileSystem{
+	imageService := image.NewService(&image.FileSystem{
 		Staging:  "/tmp/remark42/images.staging",
 		Location: "/tmp/remark42/images",
-		MaxSize:  2000,
-	}
-	imageService.TTL = 100 * time.Millisecond
+	}, image.ServiceParams{
+		EditDuration: 100 * time.Millisecond,
+		MaxSize:      2000,
+	})
+	defer imageService.Close(context.Background())
 
 	svc.privRest.imageService = imageService
 	svc.ImageService = imageService

backend/app/rest/api/rest_public.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bytes"
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
@@ -18,10 +19,10 @@ import (
 	R "github.com/go-pkgz/rest"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/image"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 type public struct {
@@ -30,7 +31,6 @@ type public struct {
 	readOnlyAge      int
 	commentFormatter *store.CommentFormatter
 	imageService     *image.Service
-	streamer         *Streamer
 	webRoot          string
 }
 
@@ -158,50 +158,6 @@ func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
-func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
-
-	sinceTs, err := s.parseSince(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
-		return
-	}
-
-	fn := func() steamEventFn {
-		lastTS := sinceTs
-		lastCount := 0
-
-		return func() (event string, data []byte, upd bool, err error) {
-			key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-			data, err = s.cache.Get(key, func() ([]byte, error) {
-				info, e := s.dataService.Info(locator, s.readOnlyAge)
-				if e != nil {
-					return nil, e
-				}
-				// cache update used as indication of post update. comparing lastTS for no-cache.
-				// removal won't update lastTS, count check will catch it.
-				if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
-					upd = true
-				}
-				lastTS = info.LastTS
-				lastCount = info.Count
-				return encodeJSONWithHTML(info)
-			})
-			if err != nil {
-				return "info", data, false, err
-			}
-
-			return "info", data, upd, nil
-		}
-	}
-
-	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
-	}
-}
-
 // GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
 // limited with "since" param
 func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
@@ -240,45 +196,6 @@ func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
-func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[DEBUG] get last comments stream for %s", siteID)
-
-	sinceTs, err := s.parseSince(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
-		return
-	}
-	if sinceTs.IsZero() {
-		sinceTs = time.Now()
-	}
-
-	fn := func() steamEventFn {
-		sinceTime := sinceTs
-		return func() (event string, data []byte, upd bool, err error) {
-			key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
-			data, err = s.cache.Get(key, func() ([]byte, error) {
-				comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
-				if e != nil {
-					return nil, e
-				}
-				if len(comments) > 0 {
-					sinceTime = comments[0].Timestamp
-					upd = true
-				}
-				sinceTime = time.Now()
-				return encodeJSONWithHTML(comments)
-			})
-			return "last", data, upd, err
-		}
-	}
-
-	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
-	}
-}
-
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
 func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -356,9 +273,10 @@ func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
 
 // POST /counts?site=siteID - get number of comments for posts from post body
 func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
+	const countBodyLimit int64 = 1024 * 128 // count request can be big for some site because it lists all urls
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
+	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, countBodyLimit), &posts); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
 		return
 	}
@@ -421,22 +339,8 @@ func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 // GET /picture/{user}/{id} - get picture
 func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
-
-	imgContentType := func(img string) string {
-		img = strings.ToLower(img)
-		switch {
-		case strings.HasSuffix(img, ".png"):
-			return "image/png"
-		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
-			return "image/jpeg"
-		case strings.HasSuffix(img, ".gif"):
-			return "image/gif"
-		}
-		return "image/*"
-	}
-
 	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
-	imgRdr, size, err := s.imageService.Load(id)
+	img, err := s.imageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
 		return
@@ -452,16 +356,10 @@ func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer func() {
-		if e := imgRdr.Close(); e != nil {
-			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
-		}
-	}()
-
-	w.Header().Set("Content-Type", imgContentType(id))
-	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.Header().Set("Content-Type", s.imageService.ImgContentType(img))
+	w.Header().Set("Content-Length", strconv.Itoa(len(img)))
 	w.WriteHeader(http.StatusOK)
-	if _, err = io.Copy(w, imgRdr); err != nil {
+	if _, err = io.Copy(w, bytes.NewReader(img)); err != nil {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -478,7 +376,7 @@ func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
 
 // GET /robots.txt
 func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
-	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/user",
 		"/img", "/avatar", "/picture"}
 	for i := range allowed {
 		allowed[i] = "Allow: /api/v1" + allowed[i]
@@ -502,13 +400,13 @@ func (s *public) applyView(comments []store.Comment, view string) []store.Commen
 }
 
 func (s *public) parseSince(r *http.Request) (time.Time, error) {
-	sinceTs := time.Time{}
+	sinceTS := time.Time{}
 	if since := r.URL.Query().Get("since"); since != "" {
 		unixTS, e := strconv.ParseInt(since, 10, 64)
 		if e != nil {
 			return time.Time{}, errors.Wrap(e, "can't translate since parameter")
 		}
-		sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+		sinceTS = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
 	}
-	return sinceTs, nil
+	return sinceTS, nil
 }

backend/app/rest/api/rest_public_test.go

@@ -1,14 +1,11 @@
 package api
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -17,8 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -39,10 +36,12 @@ func TestRest_Preview(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
 
 	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
@@ -69,7 +68,35 @@ BKT
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
+	assert.Equal(t,
+		`<h1>h1</h1>
+<pre class="chroma">func TestRest_Preview(t *testing.T) {
+srv, ts := prep(t)
+  require.NotNil(t, srv)
+}
+</pre>`,
+		string(b))
+}
+
+func TestRest_PreviewCode(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	text := `BKTgo
+func main(aa string) int {return 0}
+BKT
+`
+	text = strings.Replace(text, "BKT", "```", -1)
+	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+	j = strings.Replace(j, "\n", "\\n", -1)
+
+	resp, err := post(t, ts.URL+"/api/v1/preview", j)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, `<pre class="chroma"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+</pre>`, string(b))
 }
 
 func TestRest_Find(t *testing.T) {
@@ -183,8 +210,9 @@ func TestRest_FindReadOnly(t *testing.T) {
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
+	resp, err := client.Do(req)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 
 	tree := service.Tree{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
@@ -417,16 +445,18 @@ func TestRest_Counts(t *testing.T) {
 
 	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 
 	j := []store.PostInfo{}
 	err = json.Unmarshal(body, &j)
 	assert.NoError(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
-		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
+		{URL: "https://radio-t.com/blah2", Count: 2}}, j)
 
 	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
 	require.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
 }
 
 func TestRest_List(t *testing.T) {
@@ -498,15 +528,15 @@ func TestRest_Config(t *testing.T) {
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
 	assert.NoError(t, err)
-	assert.Equal(t, 300., j["edit_duration"])
-	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
+	assert.Equal(t, 300.0, j["edit_duration"])
+	assert.EqualValues(t, []interface{}{"a1", "a2"}, j["admins"])
 	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
-	assert.Equal(t, 4000., j["max_comment_size"])
-	assert.Equal(t, -5., j["low_score"])
-	assert.Equal(t, -10., j["critical_score"])
+	assert.Equal(t, 4000.0, j["max_comment_size"])
+	assert.Equal(t, -5.0, j["low_score"])
+	assert.Equal(t, -10.0, j["critical_score"])
 	assert.False(t, j["positive_score"].(bool))
-	assert.Equal(t, 10., j["readonly_age"])
-	assert.Equal(t, 10000., j["max_image_size"])
+	assert.Equal(t, 10.0, j["readonly_age"])
+	assert.Equal(t, 10000.0, j["max_image_size"])
 	assert.Equal(t, true, j["emoji_enabled"].(bool))
 }
 
@@ -518,11 +548,11 @@ func TestRest_Info(t *testing.T) {
 
 	user := store.User{ID: "user1", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
 	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
 
 	_, err := srv.DataService.Create(c1)
 	require.NoError(t, err, "%+v", err)
@@ -538,7 +568,7 @@ func TestRest_Info(t *testing.T) {
 	err = json.Unmarshal([]byte(body), &info)
 	assert.NoError(t, err)
 	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
-		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+		FirstTS: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
 	assert.Equal(t, exp, info)
 
 	_, code = get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah-no")
@@ -547,145 +577,6 @@ func TestRest_Info(t *testing.T) {
 	assert.Equal(t, 400, code)
 }
 
-func TestRest_InfoStream(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 800 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 0; i < 10; i++ {
-			time.Sleep(10 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	<-done
-
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
-	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
-	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
-
-	_, code = get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah123")
-	assert.Equal(t, 500, code)
-}
-
-func TestRest_InfoStreamTooMany(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 10
-
-	postComment(t, ts.URL)
-
-	var errsCount int32
-	wg := sync.WaitGroup{}
-	wg.Add(20)
-	for i := 0; i < 20; i++ {
-		go func() {
-			_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
-			if code == 429 {
-				atomic.AddInt32(&errsCount, 1)
-			}
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
-}
-
-func TestRest_InfoStreamTimeout(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	st := time.Now()
-	_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
-}
-
-func TestRest_InfoStreamCancel(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 1500 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 0; i < 5; i++ {
-			time.Sleep(300 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1", nil)
-	require.NoError(t, err)
-	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
-	defer cancel()
-	req = req.WithContext(ctx)
-	r, err := client.Do(req)
-	require.NoError(t, err)
-	defer r.Body.Close()
-	<-ctx.Done()
-	<-done
-	body, err := ioutil.ReadAll(r.Body)
-	require.EqualError(t, err, "context deadline exceeded")
-	assert.Equal(t, 200, r.StatusCode)
-
-	recs := strings.Count(string(body), "data:")
-	require.Equal(t, 1, recs, "should have 1 event:\n", string(body))
-	assert.Contains(t, string(body), `"count":2`)
-}
-
-func TestRest_InfoStreamSince(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 900 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 0; i < 10; i++ {
-			time.Sleep(15 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1&since=12345678")
-	assert.Equal(t, 200, code)
-	<-done
-	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
-	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
-}
-
 func TestRest_Robots(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -694,170 +585,6 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
-}
-
-func TestRest_LastCommentsStream(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	defer teardown()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 1; i < 10; i++ {
-			time.Sleep(100 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
-	require.NoError(t, err)
-	r, err := client.Do(req)
-	require.NoError(t, err)
-	defer r.Body.Close()
-	<-done
-	body, err := ioutil.ReadAll(r.Body)
-	require.NoError(t, err)
-	assert.Equal(t, 200, r.StatusCode)
-
-	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
-	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
-
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	require.Equal(t, 9*3, len(recs), "9 events")
-	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
-}
-
-func TestRest_LastCommentsStreamTimeout(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	st := time.Now()
-	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
-	assert.Equal(t, 200, code)
-	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
-}
-
-func TestRest_LastCommentsStreamCancel(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	defer teardown()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 1; i < 10; i++ {
-			time.Sleep(100 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
-	require.NoError(t, err)
-	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
-	defer cancel()
-	req = req.WithContext(ctx)
-	r, err := client.Do(req)
-	require.NoError(t, err)
-	<-done
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	require.EqualError(t, err, "context deadline exceeded")
-	assert.Equal(t, 200, r.StatusCode)
-
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	assert.True(t, len(recs) < 30, "less 10 events")
-}
-
-func TestRest_LastCommentsStreamTooMany(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	defer teardown()
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 10
-
-	postComment(t, ts.URL)
-
-	var errsCount int32
-	wg := sync.WaitGroup{}
-	wg.Add(20)
-	for i := 0; i < 20; i++ {
-		go func() {
-			_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
-			if code == 429 {
-				atomic.AddInt32(&errsCount, 1)
-			}
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
-
-	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
-	assert.Equal(t, 200, code, "all streams closed, good to go again")
-}
-
-func TestRest_LastCommentsStreamSince(t *testing.T) {
-	ts, srv, teardown := startupT(t)
-	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
-	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
-	srv.pubRest.streamer.MaxActive = 100
-
-	postComment(t, ts.URL)
-
-	defer teardown()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for i := 1; i < 10; i++ {
-			time.Sleep(50 * time.Millisecond)
-			postComment(t, ts.URL)
-		}
-	}()
-
-	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42&since=123456", nil)
-	require.NoError(t, err)
-	r, err := client.Do(req)
-	require.NoError(t, err)
-	<-done
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	require.NoError(t, err)
-	assert.Equal(t, 200, r.StatusCode)
-
-	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
-
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	require.Equal(t, 10*3, len(recs), "should be 10 events, including first record:\n", recs)
-}
-
-func postComment(t *testing.T, url string) {
-	resp, err := post(t, url+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
-	require.NoError(t, err)
-	b, err := ioutil.ReadAll(resp.Body)
-	require.NoError(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/user\nAllow: /api/v1/img\n"+
+		"Allow: /api/v1/avatar\nAllow: /api/v1/picture\n", body)
 }

backend/app/rest/api/rest_test.go

@@ -17,25 +17,25 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
 	cache "github.com/go-pkgz/lcw"
-	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
+	"go.uber.org/goleak"
 
-	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/notify"
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/rest/proxy"
-	"github.com/umputun/remark/backend/app/store"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/image"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/migrator"
+	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
+	"github.com/umputun/remark42/backend/app/store"
+	adminstore "github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
@@ -50,12 +50,12 @@ func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	testHtmlName := "test-remark.html"
-	testHTMLFile := os.TempDir() + "/" + testHtmlName
+	testHTMLName := "test-remark.html"
+	testHTMLFile := os.TempDir() + "/" + testHTMLName
 	err := ioutil.WriteFile(testHTMLFile, []byte("some html"), 0700)
 	assert.NoError(t, err)
 
-	body, code := get(t, ts.URL+"/web/"+testHtmlName)
+	body, code := get(t, ts.URL+"/web/"+testHTMLName)
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "some html", body)
 	_ = os.Remove(testHTMLFile)
@@ -100,11 +100,11 @@ func TestRest_Shutdown(t *testing.T) {
 func TestRest_filterComments(t *testing.T) {
 	user := store.User{ID: "user1", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
 	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
 
 	r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool {
 		return c.Text == "test test #1" || c.Text == "test test #3"
@@ -211,14 +211,17 @@ func TestRest_rejectAnonUser(t *testing.T) {
 
 	resp, err := http.Get(ts.URL)
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
 
 	resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
 
 	resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
 }
 
@@ -234,6 +237,7 @@ func Test_URLKey(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -259,6 +263,7 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -286,6 +291,7 @@ func TestRest_parseError(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
 			res := parseError(tt.err, rest.ErrInternal)
 			assert.Equal(t, tt.res, res)
@@ -309,6 +315,7 @@ func TestRest_cacheControl(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			req := httptest.NewRequest("GET", tt.url, nil)
 			w := httptest.NewRecorder()
@@ -319,35 +326,67 @@ func TestRest_cacheControl(t *testing.T) {
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
 			t.Logf("%+v", resp.Header)
 			assert.Equal(t, `"`+tt.etag+`"`, resp.Header.Get("Etag"))
-			assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds())), resp.Header.Get("Cache-Control"))
+			assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds()))+", no-cache", resp.Header.Get("Cache-Control"))
 
 		})
 	}
 
 }
 
-func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
-	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+func TestRest_frameAncestors(t *testing.T) {
 
-	tmp := os.TempDir()
-	var testDb string
-	// pick a file name which is not in use for sure
+	tbl := []struct {
+		hosts  []string
+		header string
+	}{
+		{[]string{"http://example.com"}, "frame-ancestors http://example.com;"},
+		{[]string{}, ""},
+		{[]string{"http://example.com", "http://example2.com"}, "frame-ancestors http://example.com http://example2.com;"},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := httptest.NewRequest("GET", "http://example.com", nil)
+			w := httptest.NewRecorder()
+
+			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h.ServeHTTP(w, req)
+			resp := w.Result()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			t.Logf("%+v", resp.Header)
+			assert.Equal(t, tt.header, resp.Header.Get("Content-Security-Policy"))
+
+		})
+	}
+
+}
+
+// randomPath pick a file or folder name which is not in use for sure
+func randomPath(tempDir, basename, suffix string) (string, error) {
 	for i := 0; i < 10; i++ {
-		testDb = fmt.Sprintf("/%s/test-remark-%d.db", tmp, rand.Int31())
-		_, err := os.Stat(testDb)
+		fname := fmt.Sprintf("/%s/%s-%d%s", tempDir, basename, rand.Int31(), suffix)
+		fmt.Printf("fname %q", fname)
+		_, err := os.Stat(fname)
 		if err != nil {
-			break
+			return fname, nil
 		}
 	}
+	return "", errors.New("cannot create temp file")
+}
+
+func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	tmp := os.TempDir()
+	testDB, err := randomPath(tmp, "test-remark", ".db")
+	require.NoError(t, err)
+
 	_ = os.RemoveAll(tmp + "/ava-remark42")
 	_ = os.RemoveAll(tmp + "/pics-remark42")
 
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "remark42"})
 	require.NoError(t, err)
 
-	cacheBackend, err := cache.NewExpirableCache()
-	require.NoError(t, err)
-	memCache := cache.NewScache(cacheBackend)
+	memCache := cache.NewScache(cache.NewNopCache())
 
 	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
@@ -365,21 +404,20 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
+			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
 		WebRoot:   tmp,
 		RemarkURL: "https://demo.remark42.com",
-		ImageService: &image.Service{
-			Store: &image.FileSystem{
-				Location:   tmp + "/pics-remark42",
-				Partitions: 100,
-				MaxSize:    10000,
-				Staging:    tmp + "/pics-remark42/staging",
-			},
-			TTL: time.Millisecond * 100,
-		},
+		ImageService: image.NewService(&image.FileSystem{
+			Location:   tmp + "/pics-remark42",
+			Partitions: 100,
+			Staging:    tmp + "/pics-remark42/staging",
+		}, image.ServiceParams{
+			EditDuration: 100 * time.Millisecond,
+			MaxSize:      10000,
+		}),
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -388,15 +426,10 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
-			UrlMapperMaker:    migrator.NewUrlMapper,
+			URLMapperMaker:    migrator.NewURLMapper,
 			Cache:             memCache,
 			KeyStore:          astore,
 		},
-		Streamer: &Streamer{
-			Refresh:   100 * time.Millisecond,
-			TimeOut:   5 * time.Second,
-			MaxActive: 100,
-		},
 		NotifyService: notify.NopService,
 		EmojiEnabled:  true,
 	}
@@ -407,7 +440,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 	teardown = func() {
 		ts.Close()
 		require.NoError(t, srv.DataService.Close())
-		_ = os.Remove(testDb)
+		_ = os.Remove(testDB)
 		_ = os.RemoveAll(tmp + "/ava-remark42")
 		_ = os.RemoveAll(tmp + "/pics-remark42")
 	}
@@ -429,19 +462,19 @@ func fakeAuth(next http.Handler) http.Handler {
 	return http.HandlerFunc(fn)
 }
 
-func get(t *testing.T, url string) (string, int) {
+func get(t *testing.T, url string) (response string, statusCode int) {
 	r, err := http.Get(url)
 	require.NoError(t, err)
-	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
 	require.NoError(t, err)
+	require.NoError(t, r.Body.Close())
 	return string(body), r.StatusCode
 }
 
-func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
+func sendReq(_ *testing.T, r *http.Request, tkn string) (*http.Response, error) {
 	client := http.Client{Timeout: 5 * time.Second}
-	if token != "" {
-		r.Header.Set("X-JWT", token)
+	if tkn != "" {
+		r.Header.Set("X-JWT", tkn)
 	}
 	return client.Do(r)
 }
@@ -453,25 +486,25 @@ func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	req.Header.Add("X-JWT", devToken)
 	r, err := client.Do(req)
 	require.NoError(t, err)
-	defer r.Body.Close()
 	b, err := ioutil.ReadAll(r.Body)
 	assert.NoError(t, err)
+	require.NoError(t, r.Body.Close())
 	return string(b), r.StatusCode
 }
 
-func getWithAdminAuth(t *testing.T, url string) (string, int) {
+func getWithAdminAuth(t *testing.T, url string) (response string, statusCode int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	r, err := client.Do(req)
 	require.NoError(t, err)
-	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
 	assert.NoError(t, err)
+	require.NoError(t, r.Body.Close())
 	return string(body), r.StatusCode
 }
-func post(t *testing.T, url string, body string) (*http.Response, error) {
+func post(t *testing.T, url, body string) (*http.Response, error) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
 	assert.NoError(t, err)
@@ -491,6 +524,7 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, resp.Body.Close())
 	require.NoError(t, err)
 
 	crResp := R.JSON{}
@@ -503,10 +537,12 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 func requireAdminOnly(t *testing.T, req *http.Request) {
 	resp, err := sendReq(t, req, "") // no-auth user
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 401, resp.StatusCode)
 
 	resp, err = sendReq(t, req, devToken) // non-admin user
 	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, 403, resp.StatusCode)
 }
 
@@ -532,3 +568,7 @@ func waitForHTTPSServerStart(port int) {
 		}
 	}
 }
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}

backend/app/rest/api/rss.go

@@ -10,8 +10,8 @@ import (
 	"github.com/gorilla/feeds"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 type rss struct {

backend/app/rest/api/rss_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestServer_RssPost(t *testing.T) {
@@ -279,7 +279,7 @@ func waitOnSecChange() {
 }
 
 // clean formatting, i.e. multiple spaces, \t, \n
-func cleanRssFormatting(expected, actual string) (string, string) {
+func cleanRssFormatting(expected, actual string) (cleanExp, cleanAct string) {
 	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
 
 	expected = strings.Replace(expected, "\n", " ", -1)

backend/app/rest/api/stream.go

@@ -1,103 +0,0 @@
-package api
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"sync/atomic"
-	"time"
-
-	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
-)
-
-// Streamer creates endless stream of \n separated json records send to remote client
-type Streamer struct {
-	TimeOut     time.Duration
-	Refresh     time.Duration
-	MaxActive   int32
-	activeCount int32
-}
-
-type steamEventFn func() (event string, data []byte, upd bool, err error)
-
-type steamEventResp struct {
-	data  []byte
-	event string
-	err   error
-}
-
-// Activate starts blocking function streaming update created by eventFn to ResponseWriter
-// canceled on context or inactivity timeout
-// note: eventFn is a closure needed to allow state management inside eventFn
-func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
-	updCh := s.eventsCh(ctx, eventFn())
-
-	count := atomic.AddInt32(&s.activeCount, 1)
-	defer atomic.AddInt32(&s.activeCount, -1)
-	if count > s.MaxActive {
-		return errors.New("too many streams")
-	}
-
-	if ww, ok := w.(http.ResponseWriter); ok {
-		ww.Header().Set("Content-Type", "text/event-stream")
-		ww.Header().Set("Connection", "keep-alive")
-		ww.Header().Set("Cache-Control", "no-cache")
-	}
-
-	for {
-		select {
-		case <-ctx.Done(): // request closed by remote client
-			log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
-			return nil
-		case <-time.After(s.TimeOut): // request closed by timeout
-			log.Printf("[DEBUG] stream closed due to timeout")
-			return nil
-		case resp, ok := <-updCh: // new update
-			if !ok { // closed updCh
-				return nil
-			}
-			if resp.err != nil {
-				return resp.err
-			}
-
-			// make server-sent event record
-			// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
-			if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
-				return errors.Wrap(e, "send to stream failed")
-			}
-			if fw, okFlush := w.(http.Flusher); okFlush {
-				fw.Flush()
-			}
-		}
-	}
-}
-
-// populate updates to chan, break on context close
-func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
-	ch := make(chan steamEventResp)
-	go func() {
-		tick := time.NewTicker(s.Refresh)
-		defer func() {
-			close(ch)
-			tick.Stop()
-		}()
-		for {
-			select {
-			case <-ctx.Done(): // request closed by remote client
-				return
-			case <-tick.C:
-				event, resp, upd, err := fn()
-				if err != nil {
-					ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
-					return
-				}
-				if upd {
-					ch <- steamEventResp{event: event, data: resp, err: nil}
-				}
-			}
-		}
-	}()
-	return ch
-}

backend/app/rest/api/stream_test.go

@@ -1,61 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestStream_Timeout(t *testing.T) {
-	s := Streamer{
-		Refresh:   10 * time.Millisecond,
-		TimeOut:   100 * time.Millisecond,
-		MaxActive: 10,
-	}
-
-	eventFn := func() steamEventFn {
-		n := 0
-		return func() (event string, data []byte, upd bool, err error) {
-			n++
-			if n%2 == 0 || n > 10 {
-				return "test", nil, false, nil
-			}
-			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
-		}
-	}
-
-	buf := bytes.Buffer{}
-	err := s.Activate(context.Background(), eventFn, &buf)
-	assert.NoError(t, err)
-	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
-}
-
-func TestStream_Cancel(t *testing.T) {
-	s := Streamer{
-		Refresh:   10 * time.Millisecond,
-		TimeOut:   100 * time.Millisecond,
-		MaxActive: 10,
-	}
-
-	eventFn := func() steamEventFn {
-		n := 0
-		return func() (event string, data []byte, upd bool, err error) {
-			n++
-			if n%2 == 0 {
-				return "test", nil, false, nil
-			}
-			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
-		}
-	}
-
-	buf := bytes.Buffer{}
-	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
-	defer cancel()
-	err := s.Activate(ctx, eventFn, &buf)
-	assert.NoError(t, err)
-	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
-}

backend/app/rest/httperrors.go

@@ -13,6 +13,8 @@ import (
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/rest"
+
+	"github.com/umputun/remark42/backend/app/templates"
 )
 
 // All error codes for UI mapping and translation
@@ -38,21 +40,6 @@ const (
 	ErrAssetNotFound      = 18 // requested file not found
 )
 
-const errorHtml = `<!DOCTYPE html>
-<html>
-<head>
-    <meta name="viewport" content="width=device-width"/>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-</head>
-<body>
-<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
-    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
-	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">{{.Error}}: {{.Details}}.</p>
-</div>
-</body>
-</html>
-`
-
 // errTmplData store data for error message
 type errTmplData struct {
 	Error   string
@@ -61,15 +48,22 @@ type errTmplData struct {
 
 // SendErrorHTML makes html body with provided template and responds with provided http status code,
 // error code is not included in render as it is intended for UI developers and not for the users
-func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int, t templates.FileReader) {
 	// MustExecute behaves like template.Execute, but panics if an error occurs.
 	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
-		if err := tmpl.Execute(wr, data); err != nil {
+		if err = tmpl.Execute(wr, data); err != nil {
 			panic(err)
 		}
 	}
-
-	tmpl := template.Must(template.New("error").Parse(errorHtml))
+	MustRead := func(path string) string {
+		file, e := t.ReadFile(path)
+		if e != nil {
+			panic(e)
+		}
+		return string(file)
+	}
+	tmplstr := MustRead("error_response.html.tmpl")
+	tmpl := template.Must(template.New("error").Parse(tmplstr))
 	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
 	render.Status(r, httpStatusCode)
 	msg := bytes.Buffer{}

backend/app/rest/httperrors_test.go

@@ -2,6 +2,7 @@ package rest
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -10,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestSendErrorJSON(t *testing.T) {
@@ -37,12 +38,18 @@ func TestSendErrorJSON(t *testing.T) {
 	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
-func TestSendErrorHTML(t *testing.T) {
+type MockFS struct{}
 
+func (fs *MockFS) ReadFile(path string) ([]byte, error) {
+	return []byte(fmt.Sprintf("{{.Error}}{{.Details}} %s", path)), nil
+}
+
+func TestSendErrorHTML(t *testing.T) {
+	fs := &MockFS{}
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987)
+			SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987, fs)
 			return
 		}
 		w.WriteHeader(404)

backend/app/rest/proxy/image.go

@@ -3,13 +3,10 @@ package proxy
 import (
 	"bytes"
 	"context"
-	"crypto/sha1" // nolint
 	"encoding/base64"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -18,8 +15,8 @@ import (
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store/image"
 )
 
 // Image extracts image src from comment's html and provides proxy for them
@@ -84,15 +81,6 @@ func (p Image) replace(commentHTML string, imgs []string) string {
 
 // Handler returns http handler respond to proxied request
 func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
-	if !p.HTTP2HTTPS && !p.CacheExternal {
-		// TODO: we might need to find a better way to handle it. If admin enables caching/proxy and disables it later on
-		// all comments that got converted will lose their images. We can't just return a redirect (it will open an ability
-		// to redirect anywhere). We can probably continue proxying these images (but need to make sure this behavior is
-		// documented) or, better, provide a way to migrate back converted comments.
-		http.Error(w, "none of the proxy features are enabled", http.StatusNotImplemented)
-		return
-	}
-
 	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
@@ -100,40 +88,24 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	imgURL := string(src)
-	var imgReader io.ReadCloser
-	imgID, err := cachedImgID(imgURL)
+	var img []byte
+	imgID, err := image.CachedImgID(imgURL)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse image url "+imgURL, rest.ErrAssetNotFound)
 		return
 	}
-	if p.CacheExternal {
-		imgReader, _, err = p.ImageService.Load(imgID)
-		if err != nil {
-			imgReader = nil
-		}
-	}
-	if imgReader == nil {
-		imgReader, err = p.downloadImage(context.Background(), imgURL)
+	// try to load from cache for case it was saved when CacheExternal was enabled
+	img, _ = p.ImageService.Load(imgID)
+	if img == nil {
+		img, err = p.downloadImage(context.Background(), imgURL)
 		if err != nil {
 			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
 			return
 		}
 		if p.CacheExternal {
-			var buf bytes.Buffer
-			// We need to duplicate data into a new buffer because `cacheImage` would read provider Reader
-			// and we would need another one to read data for response
-			p.cacheImage(io.TeeReader(imgReader, &buf), imgID)
-			if err := imgReader.Close(); err != nil {
-				log.Printf("[WARN] can't close image reader, %s", err)
-			}
-			imgReader = ioutil.NopCloser(&buf)
+			p.cacheImage(bytes.NewReader(img), imgID)
 		}
 	}
-	defer func() {
-		if e := imgReader.Close(); e != nil {
-			log.Printf("[WARN] can't close image reader, %s", e)
-		}
-	}()
 
 	// enforce client-side caching
 	etag := `"` + r.URL.Query().Get("src") + `"`
@@ -146,8 +118,8 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	w.Header().Add("Content-Type", "image/*")
-	_, err = io.Copy(w, imgReader)
+	w.Header().Add("Content-Type", p.ImageService.ImgContentType(img))
+	_, err = io.Copy(w, bytes.NewReader(img))
 	if err != nil {
 		log.Printf("[WARN] can't copy image stream, %s", err)
 	}
@@ -155,18 +127,14 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 
 // cache image from provided Reader using given ID
 func (p Image) cacheImage(r io.Reader, imgID string) {
-	id, err := p.ImageService.SaveWithID(imgID, r)
+	err := p.ImageService.SaveWithID(imgID, r)
 	if err != nil {
 		log.Printf("[WARN] unable to save image to the storage: %+v", err)
 	}
-	// In the future we can do something smarter than just committing everything (eg, some kind of LFU/LRU)
-	if err := p.ImageService.Commit(id); err != nil {
-		log.Printf("[WARN] unable to commit image %s", imgID)
-	}
 }
 
-// download an image. Returns a Reader which has to be closed by a caller
-func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser, error) {
+// download an image.
+func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error) {
 	log.Printf("[DEBUG] downloading image %s", imgURL)
 
 	timeout := 60 * time.Second // default
@@ -189,8 +157,7 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser,
 		return e
 	})
 	if err != nil {
-		log.Print(err.Error())
-		return nil, err
+		return nil, errors.Wrapf(err, "can't download image %s", imgURL)
 	}
 
 	if resp.StatusCode != http.StatusOK {
@@ -201,22 +168,5 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser,
 	if err != nil {
 		return nil, errors.Errorf("unable to read image body")
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(imgData)), nil
-}
-
-func sha1Str(s string) string {
-	return fmt.Sprintf("%x", sha1.Sum([]byte(s))) // nolint
-}
-
-// generates ID for a cached image.
-// ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
-// <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
-// <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
-//                              (as accurate as deduplication based on potentially mutable url can be)
-func cachedImgID(imgURL string) (string, error) {
-	parsedURL, err := url.Parse(imgURL)
-	if err != nil {
-		return "", errors.Wrapf(err, "can parse url %s", imgURL)
-	}
-	return fmt.Sprintf("cached_images/%s-%s", sha1Str(parsedURL.Hostname()), sha1Str(imgURL)), nil
+	return imgData, nil
 }

backend/app/rest/proxy/image_test.go

@@ -1,7 +1,6 @@
 package proxy
 
 import (
-	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -16,11 +15,38 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store/image"
+
+	"github.com/umputun/remark42/backend/app/store/image"
 )
 
-func TestPicture_Extract(t *testing.T) {
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestImage_Extract(t *testing.T) {
 	tbl := []struct {
 		inp string
 		res []string
@@ -53,6 +79,7 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{HTTP2HTTPS: true}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			res, err := img.extract(tt.inp, func(src string) bool { return strings.HasPrefix(src, "http://") })
 			assert.NoError(t, err)
@@ -61,7 +88,7 @@ func TestPicture_Extract(t *testing.T) {
 	}
 }
 
-func TestPicture_Replace(t *testing.T) {
+func TestImage_Replace(t *testing.T) {
 	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
 		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
@@ -69,112 +96,164 @@ func TestPicture_Replace(t *testing.T) {
 }
 
 func TestImage_Routes(t *testing.T) {
-	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Times(2).Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 }
 
-func TestImage_Routes_CachingImage(t *testing.T) {
+func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+}
+
+func TestImage_RoutesCachingImage(t *testing.T) {
 	imageStore := image.MockStore{}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
 		RoutePath:     "/api/v1/proxy",
-		ImageService:  &image.Service{Store: &imageStore},
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{MaxSize: 1500}),
 	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	imgURL := httpSrv.URL + "/image/img1.png"
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
 
-	imageStore.On("Load", mock.Anything).Once().Return(nil, int64(0), nil)
-	imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Run(func(args mock.Arguments) { _, _ = ioutil.ReadAll(args.Get(1).(io.Reader)) }).Return("", nil)
-	imageStore.On("Commit", mock.Anything).Once().Return(nil)
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+	imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
 	imageStore.AssertCalled(t, "Load", mock.Anything)
-	imageStore.AssertCalled(t, "SaveWithID", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+sha1Str(imgURL), mock.Anything)
-	imageStore.AssertCalled(t, "Commit", mock.Anything)
+	imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
 }
 
-func TestImage_Routes_Using_Cachded_Image(t *testing.T) {
+func TestImage_RoutesUsingCachedImage(t *testing.T) {
 	imageStore := image.MockStore{}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
 		RoutePath:     "/api/v1/proxy",
-		ImageService:  &image.Service{Store: &imageStore},
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{}),
 	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
-	// In order to validate that cached data is used cache "will return" some other data from what http server would
-	imageReader := ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("%256s", "X"))))
-	imageStore.On("Load", mock.Anything).Once().Return(imageReader, int64(256), nil)
+	// In order to validate that cached data used cache "will return" some other data from what http server would
+	testImage := []byte(fmt.Sprintf("%256s", "X"))
+	imageStore.On("Load", mock.Anything).Once().Return(testImage, nil)
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "256", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+	assert.Equal(t, "text/plain; charset=utf-8", resp.Header["Content-Type"][0],
+		"if you save text you receive text/plain in response, that's only fair option you got")
 
 	imageStore.AssertCalled(t, "Load", mock.Anything)
 }
 
 func TestImage_RoutesTimedOut(t *testing.T) {
-	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		Timeout:      50 * time.Millisecond,
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
-	httpSrv := imgHTTPServer(t)
+	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, resp.Body.Close())
 	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
 }
 
-func TestPicture_Convert_ProxyMode(t *testing.T) {
+func TestImage_ConvertProxyMode(t *testing.T) {
 	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
@@ -191,7 +270,7 @@ func TestPicture_Convert_ProxyMode(t *testing.T) {
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
 }
 
-func TestPicture_Convert_CachingMode(t *testing.T) {
+func TestImage_ConvertCachingMode(t *testing.T) {
 	img := Image{CacheExternal: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
@@ -206,19 +285,19 @@ func TestPicture_Convert_CachingMode(t *testing.T) {
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/>`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/>`, r)
 
-	// both Caching and Proxy are enabled
+	// both Caching and Proxy enabled
 	img = Image{CacheExternal: true, HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 }
 
-func imgHTTPServer(t *testing.T) *httptest.Server {
+func imgHTTPTestsServer(t *testing.T) *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/image/img1.png" {
 			t.Log("http img request", r.URL)
-			w.Header().Add("Content-Length", "123")
+			w.Header().Add("Content-Length", "1462")
 			w.Header().Add("Content-Type", "image/png")
-			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write(gopherPNGBytes())
 			assert.NoError(t, err)
 			return
 		}

backend/app/rest/user.go

@@ -6,7 +6,7 @@ import (
 	"github.com/go-pkgz/auth/token"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // MustGetUserInfo fails if can't extract user data from the request.

backend/app/rest/user_test.go

@@ -5,7 +5,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/backend/app/store"
+
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestUser_GetUserInfo(t *testing.T) {
@@ -20,7 +21,7 @@ func TestUser_GetUserInfo(t *testing.T) {
 	assert.Equal(t, store.User{Name: "test", ID: "id", SiteID: "test"}, u)
 }
 
-func TestUSer_MustGetUserInfo(t *testing.T) {
+func TestUser_MustGetUserInfo(t *testing.T) {
 	defer func() {
 		if r := recover(); r != nil {
 			t.Log("recovered from panic")

backend/app/store/admin/admin.go

@@ -10,7 +10,7 @@ import (
 
 // Store defines interface returning admins info for given site
 type Store interface {
-	Key() (key string, err error)
+	Key(siteID string) (key string, err error)
 	Admins(siteID string) (ids []string, err error)
 	Email(siteID string) (email string, err error)
 	Enabled(siteID string) (ok bool, err error)
@@ -37,9 +37,9 @@ type StaticStore struct {
 }
 
 // NewStaticStore makes StaticStore instance with given key
-func NewStaticStore(key string, sites []string, admins []string, email string) *StaticStore {
-	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
-	return &StaticStore{key: key, sites: sites, admins: admins, email: email}
+func NewStaticStore(key string, sites, adminIDs []string, email string) *StaticStore {
+	log.Printf("[DEBUG] admin users %+v, email %s", adminIDs, email)
+	return &StaticStore{key: key, sites: sites, admins: adminIDs, email: email}
 }
 
 // NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
@@ -48,14 +48,14 @@ func NewStaticKeyStore(key string) *StaticStore {
 }
 
 // Key returns static key, same for all sites
-func (s *StaticStore) Key() (key string, err error) {
+func (s *StaticStore) Key(_ string) (key string, err error) {
 	if s.key == "" {
 		return "", errors.New("empty key for static key store")
 	}
 	return s.key, nil
 }
 
-// Admins returns static list of admin's ids, the same for all sites
+// Admins returns static list of admin ids, the same for all sites
 func (s *StaticStore) Admins(string) (ids []string, err error) {
 	return s.admins, nil
 }
@@ -79,4 +79,4 @@ func (s *StaticStore) Enabled(site string) (ok bool, err error) {
 }
 
 // OnEvent doesn nothing for StaticStore
-func (s *StaticStore) OnEvent(siteID string, et EventType) error { return nil }
+func (s *StaticStore) OnEvent(_ string, _ EventType) error { return nil }

backend/app/store/admin/admin_test.go

@@ -7,9 +7,10 @@ import (
 )
 
 func TestStaticStore_Get(t *testing.T) {
-	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"}, []string{"123", "xyz"}, "aa@example.com")
+	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"},
+		[]string{"123", "xyz"}, "aa@example.com")
 
-	k, err := ks.Key()
+	k, err := ks.Key("any")
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 

backend/app/store/admin/remote.go

@@ -18,8 +18,8 @@ type RPC struct {
 }
 
 // Key returns the key, same for all sites
-func (r *RPC) Key() (key string, err error) {
-	resp, err := r.Call("admin.key")
+func (r *RPC) Key(siteID string) (key string, err error) {
+	resp, err := r.Call("admin.key", siteID)
 	if err != nil {
 		return "", err
 	}
@@ -35,7 +35,7 @@ func (r *RPC) Admins(siteID string) (ids []string, err error) {
 		return []string{}, err
 	}
 
-	if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+	if err := json.Unmarshal(*resp.Result, &ids); err != nil {
 		return []string{}, err
 	}
 	return ids, nil
@@ -48,7 +48,7 @@ func (r *RPC) Email(siteID string) (email string, err error) {
 		return "", err
 	}
 
-	if err = json.Unmarshal(*resp.Result, &email); err != nil {
+	if err := json.Unmarshal(*resp.Result, &email); err != nil {
 		return "", err
 	}
 	return email, nil
@@ -61,7 +61,7 @@ func (r *RPC) Enabled(siteID string) (ok bool, err error) {
 		return false, err
 	}
 
-	if err = json.Unmarshal(*resp.Result, &ok); err != nil {
+	if err := json.Unmarshal(*resp.Result, &ok); err != nil {
 		return false, err
 	}
 	return ok, nil

backend/app/store/admin/remote_test.go

@@ -19,15 +19,15 @@ import (
 )
 
 func TestRemote_Key(t *testing.T) {
-	ts := testServer(t, `{"method":"admin.key","id":1}`,
-		`{"result":"12345","id":1}`)
+	ts := testServer(t, `{"method":"admin.key","params":"any","id":1}`,
+		`{"result":"12345","params":"any","id":1}`)
 	defer ts.Close()
 	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	var a Store = &c
 	_ = a
 
-	res, err := c.Key()
+	res, err := c.Key("any")
 	assert.NoError(t, err)
 	assert.Equal(t, "12345", res)
 	t.Logf("%v %T", res, res)
@@ -96,6 +96,6 @@ func testServer(t *testing.T, req, resp string) *httptest.Server {
 		require.NoError(t, err)
 		assert.Equal(t, req, string(body))
 		t.Logf("req: %s", string(body))
-		fmt.Fprintf(w, resp)
+		_, _ = fmt.Fprint(w, resp)
 	}))
 }

backend/app/store/comment.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"fmt"
 	"html/template"
 	"regexp"
 	"strings"
@@ -26,6 +27,7 @@ type Comment struct {
 	Edit        *Edit                  `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
 	Pin         bool                   `json:"pin,omitempty" bson:"pin,omitempty"`
 	Deleted     bool                   `json:"delete,omitempty" bson:"delete"`
+	Imported    bool                   `json:"imported,omitempty" bson:"imported"`
 	PostTitle   string                 `json:"title,omitempty" bson:"title"`
 }
 
@@ -82,6 +84,7 @@ func (c *Comment) PrepareUntrusted() {
 	c.ID = ""                 // don't allow user to define ID, force auto-gen
 	c.Timestamp = time.Time{} // reset time, force auto-gen
 	c.Votes = make(map[string]bool)
+	c.VotedIPs = make(map[string]VotedIPInfo)
 	c.Score = 0
 	c.Edit = nil
 	c.Pin = false
@@ -94,6 +97,7 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 	c.Orig = ""
 	c.Score = 0
 	c.Votes = map[string]bool{}
+	c.VotedIPs = make(map[string]VotedIPInfo)
 	c.Edit = nil
 	c.Deleted = true
 	c.Pin = false
@@ -109,12 +113,21 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 // Sanitize clean dangerous html/js from the comment
 func (c *Comment) Sanitize() {
 	p := bluemonday.UGCPolicy()
-	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
+	p.AllowAttrs("class").Matching(regexp.MustCompile("^chroma$")).OnElements("pre")
+	// this is list of <span> tag classes which could be produced by chroma code renderer
+	// source: https://github.com/alecthomas/chroma/blob/022b6f4fc2c4aa819aac18363c8de3f70619200b/types.go#L221-L316
+	const codeSpanClassRegex = "^(chroma|ln|lnt|hl|lntable|lntd|w|err|x|esc|k|kc" +
+		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
+		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
+		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
+	p.AllowAttrs("class").Matching(regexp.MustCompile(codeSpanClassRegex)).OnElements("span")
+	p.AllowAttrs("loading").Matching(regexp.MustCompile("^(lazy|eager)$")).OnElements("img")
 	c.Text = p.Sanitize(c.Text)
 	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = c.escapeHtmlWithSome(c.User.Name)
-	c.User.Picture = p.Sanitize(c.User.Picture)
+	c.User.Name = c.escapeHTMLWithSome(c.User.Name)
+	c.User.Picture = c.SanitizeAsURL(c.User.Picture)
+	c.Locator.URL = c.SanitizeAsURL(c.Locator.URL)
 }
 
 // Snippet from comment's text
@@ -138,7 +151,20 @@ func (c *Comment) Snippet(limit int) string {
 	return string(snippet) + " ..."
 }
 
-func (c *Comment) escapeHtmlWithSome(inp string) string {
+var reHref = regexp.MustCompile(`<a\s+(?:[^>]*?\s+)?href="([^"]*)"`)
+
+// SanitizeAsURL drops dangerous code from a url.
+// It wraps input with href to trigger bluemonday sanitizer and cleans href after sanitizing done
+func (c *Comment) SanitizeAsURL(inp string) string {
+	h := fmt.Sprintf(`<a href="%s">`, inp)
+	clean := bluemonday.UGCPolicy().Sanitize(h)
+	if match := reHref.FindStringSubmatch(clean); len(match) > 1 {
+		return match[1]
+	}
+	return "" // this shouldn't happen as we build the href
+}
+
+func (c *Comment) escapeHTMLWithSome(inp string) string {
 	res := template.HTMLEscapeString(inp)
 	res = strings.Replace(res, "&#34;", "\"", -1)
 	res = strings.Replace(res, "&#39;", "'", -1)

backend/app/store/comment_test.go

@@ -14,6 +14,7 @@ func TestComment_Sanitize(t *testing.T) {
 		inp Comment
 		out Comment
 	}{
+
 		{inp: Comment{}, out: Comment{}},
 		{
 			inp: Comment{
@@ -47,6 +48,31 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah & & 123", User: User{Name: "name <> & ' ` \""}},
 			out: Comment{Text: `blah &amp; &amp; 123`, User: User{Name: "name &lt;&gt; & ' ` \""}},
 		},
+
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert('XSS1')"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert(document.domain)//"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "<script>alert()</script>"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: "%3Cscript%3Ealert%28%29%3C/script%3E"}},
+		},
+		{
+			inp: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+			out: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+		},
+		{
+			inp: Comment{Text: "<scrİpt>&lt;img src=x onerror=alert(1)&gt;",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+			out: Comment{Text: "&lt;img src=x onerror=alert(1)&gt;",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -78,6 +104,7 @@ func TestComment_PrepareUntrusted(t *testing.T) {
 	assert.Equal(t, time.Time{}, comment.Timestamp)
 	assert.Equal(t, false, comment.Deleted)
 	assert.Equal(t, make(map[string]bool), comment.Votes)
+	assert.Equal(t, make(map[string]VotedIPInfo), comment.VotedIPs)
 	assert.Equal(t, User{ID: "username"}, comment.User)
 
 }
@@ -101,6 +128,7 @@ func TestComment_SetDeleted(t *testing.T) {
 	assert.Equal(t, "", comment.Text)
 	assert.Equal(t, "", comment.Orig)
 	assert.Equal(t, map[string]bool{}, comment.Votes)
+	assert.Equal(t, map[string]VotedIPInfo{}, comment.VotedIPs)
 	assert.Equal(t, 0, comment.Score)
 	assert.True(t, comment.Deleted)
 	assert.Nil(t, comment.Edit)
@@ -127,6 +155,7 @@ func TestComment_SetDeletedHard(t *testing.T) {
 	assert.Equal(t, "", comment.Text)
 	assert.Equal(t, "", comment.Orig)
 	assert.Equal(t, map[string]bool{}, comment.Votes)
+	assert.Equal(t, map[string]VotedIPInfo{}, comment.VotedIPs)
 	assert.Equal(t, 0, comment.Score)
 	assert.True(t, comment.Deleted)
 	assert.Nil(t, comment.Edit)
@@ -148,6 +177,7 @@ func TestComment_Snippet(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			c := Comment{Text: tt.inp}
 			out := c.Snippet(tt.limit)
@@ -155,3 +185,40 @@ func TestComment_Snippet(t *testing.T) {
 		})
 	}
 }
+
+func TestComment_sanitizeAsURL(t *testing.T) {
+
+	tbl := []struct {
+		inp, out string
+	}{
+		{
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"javascript:alert(document.domain)//",
+			"",
+		},
+		{
+			"<script>alert()</script>",
+			"%3Cscript%3Ealert%28%29%3C/script%3E",
+		},
+		{
+			"<a href=javascript:alert(document.domain)//>xxx</a>",
+			"",
+		},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		c := Comment{}
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.out, c.SanitizeAsURL(tt.inp))
+		})
+	}
+
+}

backend/app/store/engine/bolt.go

@@ -7,12 +7,12 @@ import (
 	"strings"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
@@ -55,7 +55,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
 	result := BoltDB{dbs: make(map[string]*bolt.DB)}
 	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options)
+		db, err := bolt.Open(site.FileName, 0600, &options) //nolint:gocritic //octalLiteral is OK as FileMode
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
 		}
@@ -113,8 +113,8 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
 
 		// add reference to comment to "last" bucket
 		lastBkt = tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		if err = lastBkt.Put(commentTs, ref); err != nil {
+		commentTS := []byte(comment.Timestamp.Format(tsNano))
+		if err = lastBkt.Put(commentTS, ref); err != nil {
 			return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
 		}
 
@@ -123,7 +123,7 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
 			return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
 		}
 		// put into individual user's bucket with ts as a key
-		if err = userBkt.Put(commentTs, ref); err != nil {
+		if err = userBkt.Put(commentTS, ref); err != nil {
 			return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
 		}
 
@@ -662,7 +662,7 @@ func (b *BoltDB) getUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 		value := bucket.Get([]byte(req.UserID))
 		// return no error in case of absent entry
 		if value != nil {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 			switch req.Detail {
@@ -690,7 +690,7 @@ func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 		value := bucket.Get([]byte(req.UserID))
 		// return no error in case of absent entry
 		if value != nil {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 		}
@@ -711,7 +711,7 @@ func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry,
 	}
 
 	err = bdb.Update(func(tx *bolt.Tx) error {
-		err := b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
+		err = b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
 		return errors.Wrapf(err, "failed to update detail %s for %s in %s", req.Detail, req.UserID, req.Locator.SiteID)
 	})
 
@@ -729,7 +729,7 @@ func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err e
 		var entry UserDetailEntry
 		bucket := tx.Bucket([]byte(userDetailsBucketName))
 		return bucket.ForEach(func(userID, value []byte) error {
-			if err := json.Unmarshal(value, &entry); err != nil {
+			if err = json.Unmarshal(value, &entry); err != nil {
 				return errors.Wrap(e, "failed to unmarshal entry")
 			}
 			result = append(result, entry)
@@ -797,6 +797,14 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
 		if e = b.load(postBkt, commentID, &comment); e != nil {
 			return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
+
+		if !comment.Deleted {
+			// decrement comments count for post url
+			if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
+				return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
+			}
+		}
+
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
@@ -810,11 +818,6 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
 			return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
 		return nil
 	})
 }
@@ -844,7 +847,7 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
 
 // deleteUser removes all comments and details for given user. Everything will be market as deleted
 // and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
+func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID, userID string, mode store.DeleteMode) error {
 
 	// get list of all comments outside of transaction loop
 	posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
@@ -860,6 +863,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
 	// get list of commentID for all user's comment
 	comments := []commentInfo{}
 	for _, postInfo := range posts {
+		postInfo := postInfo
 		err = bdb.View(func(tx *bolt.Tx) error {
 			postsBkt := tx.Bucket([]byte(postsBucketName))
 			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
@@ -1007,7 +1011,8 @@ func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, er
 	}
 	info.Count++
 	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, comment.Locator.URL, &info)
+	err := b.save(infoBkt, comment.Locator.URL, &info)
+	return info, err
 }
 
 func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
@@ -1023,7 +1028,7 @@ func (b *BoltDB) makeRef(comment store.Comment) []byte {
 }
 
 // parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
+func (b *BoltDB) parseRef(val []byte) (url, id string, err error) {
 	elems := strings.Split(string(val), "!!")
 	if len(elems) != 2 {
 		return "", "", errors.Errorf("invalid reference value %s", string(val))

backend/app/store/engine/bolt_test.go

@@ -6,14 +6,14 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
-var testDb = "/tmp/test-remark.db"
+var testDB = "/tmp/test-remark.db"
 
 func TestBoltDB_CreateAndFind(t *testing.T) {
 	var b, teardown = prep(t)
@@ -221,13 +221,13 @@ func TestBoltDB_FindForUser(t *testing.T) {
 }
 
 func TestBoltDB_FindForUserPagination(t *testing.T) {
-	_ = os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	_ = os.Remove(testDB)
+	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDB, SiteID: "radio-t"})
 	require.NoError(t, err)
 
 	defer func() {
 		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
+		_ = os.Remove(testDB)
 	}()
 
 	c := store.Comment{
@@ -694,6 +694,10 @@ func TestBolt_DeleteComment(t *testing.T) {
 	assert.True(t, res[0].Deleted, "marked deleted")
 	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
 
+	// repeated deletion should not decrease comments count
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
 	assert.Equal(t, "some text2", res[1].Text)
 	assert.False(t, res[1].Deleted)
 
@@ -807,10 +811,22 @@ func TestBoltAdmin_DeleteUserHard(t *testing.T) {
 	b, teardown := prep(t)
 	defer teardown()
 
-	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+
+	// soft delete one comment
+	delReq := DeleteRequest{
+		Locator:    store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID:  comments[0].ID,
+		DeleteMode: store.SoftDelete,
+	}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
 	require.NoError(t, err)
 
-	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
 	assert.NoError(t, err)
 	require.Equal(t, 2, len(comments), "2 comments with deleted info")
 	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
@@ -836,10 +852,22 @@ func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
 	b, teardown := prep(t)
 	defer teardown()
 
-	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+
+	// soft delete one comment
+	delReq := DeleteRequest{
+		Locator:    store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID:  comments[0].ID,
+		DeleteMode: store.SoftDelete,
+	}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
 	require.NoError(t, err)
 
-	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
 	assert.NoError(t, err)
 	require.Equal(t, 2, len(comments), "2 comments with deleted info")
 	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
@@ -893,9 +921,9 @@ func TestBoltDB_NewFailed(t *testing.T) {
 
 // makes new boltdb, put two records
 func prep(t *testing.T) (b *BoltDB, teardown func()) {
-	_ = os.Remove(testDb)
+	_ = os.Remove(testDB)
 
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDB, SiteID: "radio-t"})
 	assert.NoError(t, err)
 	b = boltStore
 
@@ -921,7 +949,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) {
 
 	teardown = func() {
 		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
+		_ = os.Remove(testDB)
 	}
 	return b, teardown
 }

backend/app/store/engine/engine.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
@@ -16,18 +16,21 @@ import (
 
 // Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Create(comment store.Comment) (commentID string, err error)  // create new comment, avoid dups by id
-	Update(comment store.Comment) error                          // update comment, mutable parts only
-	Get(req GetRequest) (store.Comment, error)                   // get comment by id
-	Find(req FindRequest) ([]store.Comment, error)               // find comments for locator or site
-	Info(req InfoRequest) ([]store.PostInfo, error)              // get post(s) meta info
-	Count(req FindRequest) (int, error)                          // get count for post or user
-	Delete(req DeleteRequest) error                              // Delete post(s), user, comment, user details, or everything
-	Flag(req FlagRequest) (bool, error)                          // set and get flags
-	ListFlags(req FlagRequest) ([]interface{}, error)            // get list of flagged keys, like blocked & verified user
-	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) // sets or gets single detail value, or gets all details for requested site.
-	// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
-	// and all site's details listing under the same function (and not to extend interface by two separate functions).
+	Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
+	Update(comment store.Comment) error                         // update comment, mutable parts only
+	Get(req GetRequest) (store.Comment, error)                  // get comment by id
+	Find(req FindRequest) ([]store.Comment, error)              // find comments for locator or site
+	Info(req InfoRequest) ([]store.PostInfo, error)             // get post(s) meta info
+	Count(req FindRequest) (int, error)                         // get count for post or user
+	Delete(req DeleteRequest) error                             // Delete post(s), user, comment, user details, or everything
+	Flag(req FlagRequest) (bool, error)                         // set and get flags
+	ListFlags(req FlagRequest) ([]interface{}, error)           // get list of flagged keys, like blocked & verified user
+
+	// UserDetail sets or gets single detail value, or gets all details for requested site
+	// Returns list even for single entry request is a compromise in order to have both single detail getting and setting
+	// and all site's details listing under the same function (and not to extend interface by two separate functions)
+	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error)
+
 	Close() error // close storage engine
 }
 
@@ -83,10 +86,13 @@ const (
 	Verified = Flag("verified")
 	Blocked  = Flag("blocked")
 )
+
+// All possible user details
 const (
-	// All possible user details
-	UserEmail      = UserDetail("email")
-	AllUserDetails = UserDetail("all") // used for listing and deletion requests
+	// UserEmail is a user email
+	UserEmail = UserDetail("email")
+	// AllUserDetails used for listing and deletion requests
+	AllUserDetails = UserDetail("all")
 )
 
 // FlagRequest is the input for both get/set for flags, like blocked, verified and so on