Add subscription link support to notification email verification template (#573)

* add subscription link support to notification email verification template

* always show token for email subscription in verification email

* hide SubscribeURL from users

.drone.yml

@@ -1,92 +1,103 @@
-workspace:
+kind: pipeline
-  base: /go
+name: default
-  path: src/github.com/umputun/remark
+type: docker
 
-pipeline:
+steps:
-  
+
-  build:
+  - name: build server
-    image: golang:1.12-alpine
+    image: umputun/baseimage:buildgo-latest
     commands:
       - cd backend/app
-      - go build -v ./...
+      - go build -mod=vendor
+      - echo "build completed"
 
-  docker_master:
+  - name: docker master
     image: plugins/docker
-    repo: umputun/remark42
+    settings:
-    secrets: [ docker_username, docker_password]
+      repo: umputun/remark42
-    build_args:
+      username:
-      - DRONE=${DRONE}
+        from_secret: docker_username
-      - DRONE_TAG=${DRONE_TAG}
+      password:
-      - DRONE_COMMIT=${DRONE_COMMIT}
+        from_secret: docker_password
-      - DRONE_BRANCH=${DRONE_BRANCH}
+      build_args:
-    tags:
+        - DRONE=${DRONE}
-      - ${DRONE_COMMIT_BRANCH/\//-}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
     when:
-      branch: [master, release/*]
+      branch: [master]
       event: push
 
-  docker_tag:
+  - name: docker tag
     image: plugins/docker
-    repo: umputun/remark42
+    settings:
-    secrets: [ docker_username, docker_password ]
+      repo: umputun/remark42
-    build_args:
+      username:
-      - DRONE=${DRONE}
+        from_secret: docker_username
-      - DRONE_TAG=${DRONE_TAG}
+      password:
-      - DRONE_COMMIT=${DRONE_COMMIT}
+        from_secret: docker_password
-    tags: 
+      build_args:
-      - ${DRONE_TAG}
+        - DRONE=${DRONE}
-      - latest
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+      tags:
+        - ${DRONE_TAG}
+        - latest
     when:
       event: tag
 
-  artifacts_tag:
+  - name: docker branch
     image: plugins/docker
-    dockerfile: Dockerfile.artifacts
+    settings:
-    build_args:
+      repo: umputun/remark42
-      - DRONE=${DRONE}
+      username:
-      - DRONE_TAG=${DRONE_TAG}
+        from_secret: docker_username
-      - DRONE_COMMIT=${DRONE_COMMIT}
+      password:
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
+        from_secret: docker_password
-    when:
+      build_args:
-      event: tag
+        - DRONE=${DRONE}
-
+        - DRONE_COMMIT=${DRONE_COMMIT}
-  docker_branch:
+        - DRONE_BRANCH=${DRONE_BRANCH}
-    image: plugins/docker
+      tags:
-    repo: umputun/remark42
+        - ${DRONE_COMMIT_BRANCH/\//-}
-    secrets: [ docker_username, docker_password ]
+      dry_run: true
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
     when:
       branch:
         exclude: [master, release/*]
         event: push
 
-  deploy:
+  - name: artifacts tag
-    image: appleboy/drone-ssh
+    image: plugins/docker
-    host: remark42.com
+    settings:
-    username: umputun
+      dockerfile: Dockerfile.artifacts
-    port: 22
+      build_args:
-    secrets: [ ssh_key ]
+        - DRONE=${DRONE}
-    script:
+        - DRONE_TAG=${DRONE_TAG}
-      - cd /srv/remark
+        - DRONE_COMMIT=${DRONE_COMMIT}
-      - docker-compose pull
+        - GITHUB_TOKEN=${GITHUB_TOKEN}
-      - docker-compose up -d
+    when:
+      event: tag
+
+  - name: deploy
+    image: docker.umputun.com/system/deploy-ci:master
+    commands:
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
     when:
       branch: master
       event: push
 
-  notify:
+  - name: notify
     image: drillster/drone-email
-    host: smtp.mailgun.org
+    settings:
-    port: 25
+      host: smtp.mailgun.org
-    username: email_username
+      username:
-    password: email_password
+        from_secret: email_username
-    from: drone@mg.umputun.com
+      password:
-    recipients: [ umputun@gmail.com ]
+        from_secret: email_password
-    secrets: [ email_username, email_password ]
+      from: drone@mg.umputun.dev
+      recipients: [ sys@umputun.dev ]
     when:
       status: [ changed, failure ]

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*       @umputun

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+#patreon: remark42

.github/workflows/ci-build.yml

@@ -0,0 +1,31 @@
+name: build
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: build docker image
+        run: docker build --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github .

.github/workflows/ci-test-backend.yml

@@ -0,0 +1,67 @@
+name: test_backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+      - '.golangci.yml'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+      - '.golangci.yml'
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: debug if needed
+        run: |
+          export DEBUG=${DEBUG:-false}
+          if [[ "$DEBUG" == "true" ]]; then
+            env
+          fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: test backend
+        run: |
+          export TZ="America/Chicago"
+          date
+          cd backend/app
+          go test -mod=vendor -timeout=60s -covermode=count -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          cd ../_example/memory_store
+          go test -race ./...
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.20.0
+          go get -u github.com/mattn/goveralls
+
+      - name: run backend linters
+        run: $GITHUB_WORKSPACE/golangci-lint run --config .golangci.yml ./...
+        working-directory: backend
+
+      - name: run linters for examples
+        run: $GITHUB_WORKSPACE/golangci-lint run --config ${GITHUB_WORKSPACE}/backend/.golangci.yml ./...
+        working-directory: backend/_example/memory_store
+
+      - name: submit coverage
+        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-test-frontend.yml

@@ -0,0 +1,28 @@
+name: test_frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+
+jobs:
+  frontend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions/setup-node@v1
+
+      - run: npm ci
+        working-directory: ./frontend
+
+      - run: npx run-p check lint test
+        working-directory: ./frontend

.gitignore

@@ -20,3 +20,6 @@ remark42
 /bin/
 /backend/var/
 compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor

.golangci.yml

@@ -0,0 +1,2 @@
+service:
+  golangci-lint-version: 1.19.1

.travis.yml

@@ -1,19 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      .

Dockerfile

@@ -1,17 +1,6 @@
 FROM umputun/baseimage:buildgo-latest as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
 ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
@@ -19,9 +8,10 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
+ARG BACKEND_TEST_TIMEOUT
 
 ADD backend /build/backend
-ADD .git /build/.git
+ADD .git/ /build/backend/.git/
 WORKDIR /build/backend
 
 ENV GOFLAGS="-mod=vendor"
@@ -30,31 +20,18 @@ ENV GOFLAGS="-mod=vendor"
 RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=count -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
-    else echo "skip backend test" ; fi
+        golangci-lint run --config ../.golangci.yml ./... ; \
-
+    else echo "skip backend tests and linter" ; fi
-# linters
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-        --enable=ineffassign --enable=varcheck ./... ; \
-    else echo "skip backend linters" ; fi
-
-# submit coverage to coverals if COVERALLS_TOKEN in env
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo "coverall not enabled" ; \
-    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
-    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
@@ -74,16 +51,15 @@ ARG NODE_ENV=production
 COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
 ADD frontend /srv/frontend
 RUN cd /srv/frontend && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p check lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-
+FROM umputun/baseimage:app
-FROM umputun/baseimage:app-latest
 
 WORKDIR /srv
 
-ADD entrypoint.sh /entrypoint.sh
+ADD docker-init.sh /entrypoint.sh
 ADD backend/scripts/backup.sh /usr/local/bin/backup
 ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
@@ -97,5 +73,6 @@ RUN ln -s /srv/remark42 /usr/bin/remark42
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["server"]
+COPY docker-init.sh /srv/init.sh
-ENTRYPOINT ["/entrypoint.sh"]
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -26,38 +26,44 @@ RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
-
 FROM umputun/baseimage:buildgo-latest as build-backend
 
 ARG GITHUB_TOKEN
 ENV SKIP_BACKEND_TEST=true
 
-WORKDIR /go/src/github.com/umputun/remark/backend
+RUN go get github.com/rakyll/statik
-ADD backend /go/src/github.com/umputun/remark/backend
+
-ADD README.md /go/src/github.com/umputun/remark/
+WORKDIR /build/backend
-ADD LICENSE /go/src/github.com/umputun/remark/
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+ADD .git/ /build/backend/.git/
+
 COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
-    export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
+    export WEB_ROOT=/build/backend/web && \
     sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
     sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
-    go get -v github.com/rakyll/statik && \
+    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
-    statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
+    ls -la /build/backend/app/rest/api/statik.go && \
-    ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
+    ls -la /build/backend/web/
-    ls -la /go/src/github.com/umputun/remark/backend/web/
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
     if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    echo "runs outside of drone" && version=$(/script/git-rev.sh); \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
+    export GOFLAGS="-mod=vendor" && \
     GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
-    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
 
 RUN \
     if [ -z "$DRONE_TAG" ] ; then \
@@ -67,8 +73,10 @@ RUN \
     cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
     tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
     tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
     tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
     tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
     zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
 # upload to github
@@ -93,6 +101,6 @@ RUN \
 #    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
 RUN ls -la /artifacts/*
 CMD ["sleep", "100"]

Makefile

@@ -20,7 +20,21 @@ deploy:
 	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
 	docker rm -f remark42.bin
 
-.PHONY: bin
+race_test:
+	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,9 +1,11 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# remark42 [![Build Status](https://github.com/umputun/remark/workflows/build/badge.svg)](https://github.com/umputun/remark/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+
+
 
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, GitHub and Yandex
-* Login via email 
+* Login via email
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
 * Import from Disqus and WordPress
@@ -14,7 +16,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
-* Telegram notifications
+* Telegram and email notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
@@ -48,6 +50,7 @@ For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
         - [Facebook Auth Provider](#facebook-auth-provider)
+        - [Twitter Auth Provider](#twitter-auth-provider)
         - [Yandex Auth Provider](#yandex-auth-provider)
       - [Initial import from Disqus](#initial-import-from-disqus)
       - [Initial import from WordPress](#initial-import-from-wordpress)
@@ -98,94 +101,123 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default                  | Description                                      |
+| Command line            | Environment             | Default                  | Description                                     |
-| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
+| ----------------------- | ----------------------- | ------------------------ | ----------------------------------------------- |
-| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_              |
-| secret                  | SECRET                  |                          | secret key, _required_                           |
+| secret                  | SECRET                  |                          | secret key, _required_                          |
-| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                           |
-| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `rpc`                |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                          |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                           |
-| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_         |
-| mongo.db                | MONGO_DB                |                          | mongo database                                   |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                     |
-| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                        |
-| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited     |
-| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited       |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited  |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `uri`  |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                 |
-| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                     |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
+| avatar.uri              | AVATAR_URI              | `./var/avatars`          | avatar store uri                                |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save     |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, `bolt`             |
-| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                          |
-| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                    |
-| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                      |
-| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                      |
-| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
+| image.bolt.file         | IMAGE_BOLT_FILE         | `/var/pictures.db`       | images bolt file location                       |
-| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                           |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `2400`                   | width of resized image                          |
-| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `900`                    | height of resized image                         |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                         |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                      |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                          |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                      |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                        |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                    |
-| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                          |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                      |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
+| auth.twitter.cid        | AUTH_TWITTER_CID        |                          | Twitter Consumer API Key                        |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
+| auth.twitter.csec       | AUTH_TWITTER_CSEC       |                          | Twitter Consumer API Secret key                 |
-| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                          |
-| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                      |
-| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                            |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only      |
-| auth.email.host         | AUTH_EMAIL_HOST         |                          | smtp host                                        |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                          |
-| auth.email.port         | AUTH_EMAIL_PORT         | `25`                     | smtp port                                        |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                           |
-| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                       |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                      |
-| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                    |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                   |
-| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                               |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                              |
-| auth.email.tls          | AUTH_EMAIL_TLS          | `false`                  | enable TLS                                       |
+| auth.email.template     | AUTH_EMAIL_TEMPLATE     | none (predefined)        | custom email message template file              |
-| auth.email.user         | AUTH_EMAIL_USER         |                          | smtp user name                                   |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (telegram and/or email)    |
-| auth.email.passwd       | AUTH_EMAIL_PASSWD       |                          | smtp password                                    |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                      |
-| auth.email.timeout      | AUTH_EMAIL_TIMEOUT      | `10s`                    | smtp timeout                                     |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                  |
-| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                |
-| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
+| notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
-| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
-| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
-| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| smtp.password           | SMTP_PASSWORD           |                          | SMTP password                                   |
-| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| smtp.tls                | SMTP_TLS                |                          | enable TLS for SMTP                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| smtp.timeout            | SMTP_TIMEOUT            | `10s`                    | SMTP TCP connection timeout                     |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le  |
-| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                           |
-| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                           |
-| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                            |
-| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored      |
-| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE |
-| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                            |
-| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited       |
-| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| votes-ip                | VOTES_IP                | `false`                  | restrict votes from the same ip                 |
-| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| anon-vote               | ANON_VOTE               | `false`                  | allow voting for anonymous users, require VOTES_IP to be enabled as well |
-| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| votes-ip-time           | VOTES_IP_TIME           | `5m`                     | same ip vote restriction time, `0s` - unlimited |
-| emoji                   | EMOJI                   | `false`                  | enable emoji support                             |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                             |
-| port                    | REMARK_PORT             | `8080`                   | web server port                                  |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                        |
-| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                        |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive   |
-| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_ |
-| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                     |
-| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                 |
+| image-proxy.http2https  |  IMAGE_PROXY_HTTP2HTTPS | `false`                  | enable http->https proxy for images             |
+| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false`            | enable caching external images to current image storage |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                            |
+| simple-view             | SIMPLE_VIEW             | `false`                  | minimized UI with basic info only               |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                 |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                       |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                               |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                 |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                      |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
 * _required_ parameters have to be presented in the environment or provided in command line
 
+##### Deprecated
+
+Following list of command-line options is deprecated and will be removed in 2 minor releases or 1 major release (whichever is closer)
+from the version in which they were deprecated. After remark42 version update, please check startup log once for deprecation warnings to avoid
+trouble with unrecognized command-line options in the future.
+
+<details>
+<summary>deprecated options</summary>
+
+| Command line       | Replacement   | Environment        | Replacement   | Default | Description    | Deprecation version |
+| ------------------ | ------------- | ------------------ | ------------- | ------- | -------------- | ------------------- |
+| auth.email.host    | smtp.host     | AUTH_EMAIL_HOST    | SMTP_HOST     |         | smtp host      | 1.5.0               |
+| auth.email.port    | smtp.port     | AUTH_EMAIL_PORT    | SMTP_PORT     |         | smtp port      | 1.5.0               |
+| auth.email.user    | smtp.username | AUTH_EMAIL_USER    | SMTP_USERNAME |         | smtp user name | 1.5.0               |
+| auth.email.passwd  | smtp.password | AUTH_EMAIL_PASSWD  | SMTP_PASSWORD |         | smtp password  | 1.5.0               |
+| auth.email.tls     | smtp.tls      | AUTH_EMAIL_TLS     | SMTP_TLS      | `false` | enable TLS     | 1.5.0               |
+| auth.email.timeout | smtp.timeout  | AUTH_EMAIL_TIMEOUT | SMTP_TIMEOUT  | `10s`   | smtp timeout   | 1.5.0               |
+| img-proxy          | image-proxy.http2https | IMG_PROXY | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images | 1.5.0 |
+</details>
+
 ##### Required parameters
 
 Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
-2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
+2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.remark42.com`
 3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
 
 The minimal `docker-compose.yml` has to include all required parameters:
@@ -251,6 +283,14 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
 1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
 
+##### Twitter Auth Provider
+
+1.	Create a new twitter application https://developer.twitter.com/en/apps
+1.	Fill **App name**, **Description** and **URL** of your site
+1.	In the field **Callback URLs** enter the correct url of your callback handler e.g.  domain + `/auth/twitter/callback`
+1.	Under **Key and tokens** take note of the **Consumer API Key** and **Consumer API Secret key**. Those will be used as `AUTH_TWITTER_CID` and
+ `AUTH_TWITTER_CSEC`
+
 ##### Yandex Auth Provider
 
 1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
@@ -321,6 +361,39 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
+#### Docker parameters
+
+Two parameters allow to customize docker container on the system level:
+
+- `APP_UID` - sets UID to run remark42 application in container (default=1001)
+- `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
+
+_see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
+
+example of compose:
+
+```yaml
+version: '2'
+
+services:
+    remark42:
+        image: umputun/remark42:latest
+        restart: always
+        container_name: "remark42"
+        environment:
+            - APP_UID=2000                          # runs remark42 app with non-default UID
+            - TIME_ZONE=GTC                         # sets container time to UTC
+
+            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
+            - SECRET=abcd-123456-xyz-$%^&           # secret key
+            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
+            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
+        volumes:
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
+```
+
+
 ### Setup on your website
 
 #### Comments
@@ -334,13 +407,31 @@ Add this snippet to the bottom of web page:
   var remark_config = {
     host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
-    components: ['embed'] // optional param; which components to load. default to ["embed"]
+    components: ['embed'], // optional param; which components to load. default to ["embed"]
-                          // to load all components define components as ['embed', 'last-comments', 'counter']
+                           // to load all components define components as ['embed', 'last-comments', 'counter']
-                          // available component are:
+                           // available component are:
-                          //     - 'embed': basic comments widget
+                           //     - 'embed': basic comments widget
-                          //     - 'last-comments': last comments widget, see `Last Comments` section below
+                           //     - 'last-comments': last comments widget, see `Last Comments` section below
-                          //     - 'counter': counter widget, see `Counter` section below
+                           //     - 'counter': counter widget, see `Counter` section below
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
+    url: 'PAGE_URL', // optional param; if it isn't defined
+                     // `window.location.origin + window.location.pathname` will be used,
+                     //
+                     // Note that if you use query parameters as significant part of url
+                     // (the one that actually changes content on page)
+                     // you will have to configure url manually to keep query params, as
+                     // `window.location.origin + window.location.pathname` doesn't contain query params and
+                     // hash. For example default url for `https://example/com/example-post?id=1#hash`
+                     // would be `https://example/com/example-post`.
+                     //
+                     // The problem with query params is that they often contain useless params added by
+                     // various trackers (utm params) and doesn't have defined order, so Remark treats differently
+                     // all this examples:
+                     // https://example.com/?postid=1&date=2007-02-11
+                     // https://example.com/?date=2007-02-11&postid=1
+                     // https://example.com/?date=2007-02-11&postid=1&utm_source=google
+                     //
+                     // If you deal with query parameters make sure you pass only significant part of it
+                     // in well defined order
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
@@ -446,7 +537,7 @@ You can use as many nodes like this as you need to.
 The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
+Also script can use `url` property from `remark_config` object, or `window.location.origin + window.location.pathname` if nothing else is defined.
 
 ## Build from the source
 
@@ -481,12 +572,9 @@ Frontend docker compose config by default skips running backend related tests an
 
 In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
 
-To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
+To run backend - `go run backend/app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
 It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
-`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
+`HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
-
-To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
-`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
 
 ### Frontend development
 
@@ -519,7 +607,7 @@ Developer build running by `webpack-dev-server` supports devtools for [React](ht
 
 #### Frontend guide
 
-Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
 
 ## API
 
@@ -634,7 +722,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
   ```go
   type Config struct {
-     	Version        string   `json:"version"`
+        Version        string   `json:"version"`
         EditDuration   int      `json:"edit_duration"`
         MaxCommentSize int      `json:"max_comment_size"`
         Admins         []string `json:"admins"`
@@ -682,7 +770,7 @@ event: info
 data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
 
 event: info
-data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}         
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}
 ```
 
 </details>
@@ -700,6 +788,17 @@ data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:5
 
 _returned id should be appended to load image url on caller side_
 
+### Email subscription
+
+* `GET /api/v1/email?site=site-id` - get user's email, _auth required_
+* `POST /api/v1/email/subscribe?site=site-id&address=user@example.org` -  makes confirmation token and sends it to user over email, _auth required_
+
+  Trying to subscribe same email second time will return response code `409 Conflict` and explaining error message.
+* `POST /api/v1/email/confirm?site=site-id&tkn=token` - uses provided token parameter to set email for the user, _auth required_
+
+  Setting email subscribe user for all first-level replies to his messages.
+* `DELETE /api/v1/email?site=siteID` - removes user's email, _auth required_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -712,10 +811,17 @@ _returned id should be appended to load image url on caller side_
       Until     time.Time `json:"time"`
   }
   ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
+* `GET /api/v1/admin/export?site=site-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
+* `POST /api/v1/admin/import?site=site-id` - import comments from the backup, uses post body.
-* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
+* `POST /api/v1/admin/import/form?site=site-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion.
+* `POST /api/v1/admin/remap?site=site-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
+From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
+export/import chain so make backup first.
+    ```
+    http://oldsite.com* https://newsite.com*
+    http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
+    ```
+* `GET /api/v1/admin/wait?site=site-id` - wait for completion for any async migration ops (import or remap).
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
 * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
 * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.

backend/.golangci.yml

@@ -0,0 +1,22 @@
+run:
+  tests: false
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+linters:
+  enable:
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+  disable-all: true

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,22 @@
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+#ADD . /build/memory_store
+#WORKDIR /build/memory_store
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+
+FROM umputun/baseimage:app-latest
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,13 @@
+# sample store implementation 
+
+`memory_store` illustrates how to make a custom storage plugin for remark42. 
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. docker-compose -f compose-dev-memstore.yml build
+1. docker-compose -f compose-dev-memstore.yml up
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package
+. In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key() (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, errors.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", errors.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, errors.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return errors.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated += 1 // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,563 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store/engine"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.Lock()
+	defer m.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", errors.New("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.Lock()
+	defer m.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, errors.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, errors.New("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.Lock()
+	defer m.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		m.Lock()
+		defer m.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.Lock()
+			defer m.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.Lock()
+	defer m.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return errors.New("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return errors.New("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	return status, errors.Wrapf(err, "failed to set flag %+v", req)
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, errors.New("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID == comment.ID && c.Locator == comment.Locator {
+			c.Text = comment.Text
+			c.Orig = comment.Orig
+			c.Score = comment.Score
+			c.Votes = comment.Votes
+			c.Pin = comment.Pin
+			c.Deleted = comment.Deleted
+			c.User = comment.User
+			comments[i] = c
+			m.posts[comment.Locator.SiteID] = comments
+			return nil
+		}
+	}
+	return errors.New("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,784 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds []string
+	for _, x := range blockedList {
+		blockedIds = append(blockedIds, x.ID)
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		emailSet   = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,47 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: '2'
+
+services:
+
+  remark42:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42"
+    hostname: "remark42"
+    restart: always
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - BACKUP_PATH=/srv/var/backup
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: .
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,16 @@
+module github.com/umputun/remark/memory_store
+
+go 1.12
+
+require (
+	github.com/go-pkgz/jrpc v0.1.0
+	github.com/go-pkgz/lgr v0.6.3
+	github.com/jessevdk/go-flags v1.4.0
+	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.4.0
+	github.com/umputun/remark/backend v1.4.0
+)
+
+replace github.com/umputun/remark/backend => ../../
+
+replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1

backend/_example/memory_store/go.sum

@@ -0,0 +1,330 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
+github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-pkgz/auth v0.9.0/go.mod h1:8Kiold3xA9VUiYZwMyMhwBgiGImZFoL/EmtO9lSIQxo=
+github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
+github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
+github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
+github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
+github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.7.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
+github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a h1:Yu34BogBivvmu7SAzHHaB9nZWH5D1C+z3F1jyIaYZSQ=
+golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
+gopkg.in/oauth2.v3 v3.11.0/go.mod h1:ZdKwOng79bFq17t/ItHGtL8LJ2dW3tmrdJJhHmqS+uc=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/_example/memory_store/main.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/jessevdk/go-flags"
+
+	"github.com/umputun/remark/memory_store/accessor"
+	"github.com/umputun/remark/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+
+	rpcServer := jrpc.Server{
+		API:        opts.API,
+		AuthUser:   opts.AuthUser,
+		AuthPasswd: opts.AuthPasswd,
+		Version:    revision,
+		AppName:    "remark42-memory",
+		Logger:     log.Default(),
+	}
+
+	srv := server.NewRPC(dataStore, adminStore, &rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID: "remark",
+		IDs:    []string{"dev_user"},
+		Email:  "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/rpc.go

@@ -0,0 +1,234 @@
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
+// Note: this file can be used as-is in any custom jrpc plugin
+type RPC struct {
+	*jrpc.Server
+	eng engine.Interface
+	adm admin.Store
+}
+
+// NewRPC makes RPC instance and register handlers
+func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, Server: r}
+	res.addHandlers()
+	return res
+}
+
+func (s *RPC) addHandlers() {
+	// data store handlers
+	s.Group("store", jrpc.HandlersGroup{
+		"create":      s.createHndl,
+		"find":        s.findHndl,
+		"get":         s.getHndl,
+		"update":      s.updateHndl,
+		"count":       s.countHndl,
+		"info":        s.infoHndl,
+		"flag":        s.flagHndl,
+		"list_flags":  s.listFlagsHndl,
+		"user_detail": s.userDetailHndl,
+		"delete":      s.deleteHndl,
+		"close":       s.closeHndl,
+	})
+
+	// admin store handlers
+	s.Group("admin", jrpc.HandlersGroup{
+		"key":     s.admKeyHndl,
+		"admins":  s.admAdminsHndl,
+		"email":   s.admEmailHndl,
+		"enabled": s.admEnabledHndl,
+		"event":   s.admEventHndl,
+	})
+}
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	key, err := s.adm.Key()
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	ps := []interface{}{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/rpc_test.go

@@ -0,0 +1,432 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+
+	"github.com/umputun/remark/memory_store/accessor"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}
+func TestRPC_admKeyHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
+	mg := accessor.NewMemData()
+	adm := accessor.NewMemAdminStore("secret")
+	s = NewRPC(mg, adm, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+
+	admRec := accessor.AdminRec{
+		SiteID:  "test-site",
+		IDs:     []string{"id1", "id2"},
+		Email:   "admin@example.com",
+		Enabled: true,
+	}
+	adm.Set("test-site", admRec)
+
+	admRecDisabled := admRec
+	admRecDisabled.Enabled = false
+	adm.Set("test-site-disabled", admRecDisabled)
+
+	port = chooseRandomUnusedPort()
+	go func() {
+		log.Printf("%v", s.Run(port))
+	}()
+
+	waitForHTTPServerStart(port)
+
+	return s, port, func() {
+		require.NoError(t, s.Shutdown())
+	}
+}

backend/app/cmd/backup_test.go

@@ -25,13 +25,13 @@ func TestBackup_Execute(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	defer os.Remove("/tmp/remark-test.export")
 
 	data, err := ioutil.ReadFile("/tmp/remark-test.export")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
@@ -49,7 +49,7 @@ func TestBackup_ExecuteFailedStatus(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
 }
@@ -68,7 +68,7 @@ func TestBackup_ExecuteFailedWrite(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
 		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,8 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL) // nolint
+	client := http.Client{Timeout: 30 * time.Second}
+	r, err := client.Get(listURL)
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +159,8 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL) // nolint
+		client := http.Client{Timeout: 30 * time.Second}
+		r, err = client.Get(commentsURL)
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}

backend/app/cmd/cleanup_test.go

@@ -68,7 +68,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	posts, err := cmd.postsInRange("20181218", "20181219")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts))
@@ -78,7 +78,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	assert.Equal(t, 3, len(posts))
 
 	_, err = cmd.postsInRange("xxx", "yyy")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestCleanup_listComments(t *testing.T) {
@@ -91,7 +91,7 @@ func TestCleanup_listComments(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	comments, err := cmd.listComments("http://test.com/post1")
 	assert.NoError(t, err)
@@ -118,7 +118,7 @@ func TestCleanup_ExecuteSpam(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
 		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("deleted %+v", cleaned.ids)
@@ -136,7 +136,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("set titles for %+v", titledComments.ids)

backend/app/cmd/cmd.go

@@ -21,6 +21,7 @@ import (
 type CommonOptionsCommander interface {
 	SetCommon(commonOpts CommonOpts)
 	Execute(args []string) error
+	HandleDeprecatedFlags() []DeprecatedFlag
 }
 
 // CommonOpts sets externally from main, shared across all commands
@@ -30,6 +31,13 @@ type CommonOpts struct {
 	Revision     string
 }
 
+// DeprecatedFlag contains information about deprecated option
+type DeprecatedFlag struct {
+	Old           string
+	New           string
+	RemoveVersion string
+}
+
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
@@ -38,6 +46,9 @@ func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
 	c.Revision = commonOpts.Revision
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated and returns their list
+func (c *CommonOpts) HandleDeprecatedFlags() []DeprecatedFlag { return nil }
+
 // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
 type fileParser struct {
 	site string

backend/app/cmd/cmd_test.go

@@ -31,7 +31,7 @@ func TestExport_ParseFileName(t *testing.T) {
 	for i, tt := range tbl {
 		r, err := tt.p.parse(now)
 		if tt.err {
-			assert.NotNil(t, err)
+			assert.Error(t, err)
 			continue
 		}
 		assert.Equal(t, tt.res, r, "check #%d", i)

backend/app/cmd/import.go

@@ -18,7 +18,7 @@ import (
 // ImportCommand set of flags and command for import
 type ImportCommand struct {
 	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
+	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint
 	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
 	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
 	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`

backend/app/cmd/import_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
@@ -21,7 +20,7 @@ func TestImport_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -34,7 +33,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -43,7 +42,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
@@ -61,21 +60,21 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on no such file")
+	assert.Error(t, err, "fail on no such file")
-	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
+	assert.Contains(t, err.Error(), "no such file or directory")
 
 	cmd = ImportCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on connection refused")
+	assert.Error(t, err, "fail on connection refused")
-	assert.True(t, strings.Contains(err.Error(), "connection refused"))
+	assert.Contains(t, err.Error(), "connection refused")
 
 	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		log.Printf("%+v", r)
@@ -87,10 +86,10 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestImport_ExecuteTimeout(t *testing.T) {
@@ -98,7 +97,7 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 		time.Sleep(500 * time.Millisecond)
 		fmt.Fprintln(w, "some response")
@@ -112,8 +111,8 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
-	assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
+	assert.Contains(t, err.Error(), "deadline exceeded")
 }

backend/app/cmd/remap.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// RemapCommand set of flags and command for change linkage between comments to
+// different urls based on given rules (input file)
+type RemapCommand struct {
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
+	CommonOpts
+}
+
+// Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
+func (rc *RemapCommand) Execute(args []string) error {
+	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	rulesReader, err := os.Open(rc.InputFile)
+	if err != nil {
+		return errors.Wrapf(err, "cant open file %s", rc.InputFile)
+	}
+
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
+	defer cancel()
+	remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
+	req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
+	if err != nil {
+		return errors.Wrapf(err, "can't make remap request for %s", remapURL)
+	}
+	req.SetBasicAuth("admin", rc.AdminPasswd)
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", remapURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "can't get response")
+	}
+
+	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+	return nil
+}

backend/app/cmd/remap_test.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemap_Execute(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		body, err := ioutil.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(202)
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}

backend/app/cmd/restore_test.go

@@ -19,7 +19,7 @@ func TestRestore_Execute(t *testing.T) {
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "native", r.URL.Query().Get("provider"))
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -32,7 +32,7 @@ func TestRestore_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }

backend/app/cmd/server.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -14,6 +15,8 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/dgrijalva/jwt-go"
+	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
 	"github.com/kyokomi/emoji"
 	authcache "github.com/patrickmn/go-cache"
@@ -24,13 +27,12 @@ import (
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
+	cache "github.com/go-pkgz/lcw"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest/api"
 	"github.com/umputun/remark/backend/app/rest/proxy"
-	"github.com/umputun/remark/backend/app/rpc"
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
@@ -40,32 +42,38 @@ import (
 
 // ServerCommand with command line flags and env
 type ServerCommand struct {
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
+	Store      StoreGroup      `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
+	Avatar     AvatarGroup     `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
+	Cache      CacheGroup      `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
+	Admin      AdminGroup      `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
-	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Notify     NotifyGroup     `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
-	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
+	SMTP       SmtpGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
-	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Image      ImageGroup      `group:"image" namespace:"image" env-namespace:"IMAGE"`
-	Stream StreamGroup `group:"stream" namespace:"stream" env-namespace:"STREAM"`
+	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Stream     StreamGroup     `group:"stream" namespace:"stream" env-namespace:"STREAM"`
+	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
-	BackupLocation  string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	MaxBackupFiles  int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	ImageProxy      bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
+	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	MaxCommentSize  int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
-	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
-	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
-	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	EnableEmoji     bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView       bool          `long:"simpler-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
 
 	Auth struct {
 		TTL struct {
@@ -76,25 +84,33 @@ type ServerCommand struct {
 		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
 		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
 		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
 		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
 		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
 		Email     struct {
 			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
-			Host         string        `long:"host" env:"HOST" description:"smtp host"`
+			From         string        `long:"from" env:"FROM" description:"from email address"`
-			Port         int           `long:"port" env:"PORT" description:"smtp port"`
-			From         string        `long:"from" env:"FROM" description:"email's from"`
 			Subject      string        `long:"subj" env:"SUBJ" default:"remark42 confirmation" description:"email's subject"`
 			ContentType  string        `long:"content-type" env:"CONTENT_TYPE" default:"text/html" description:"content type"`
-			TLS          bool          `long:"tls" env:"TLS" description:"enable TLS"`
+			Host         string        `long:"host" env:"HOST" description:"[deprecated, use --smtp.host] SMTP host"`
-			SMTPUserName string        `long:"user" env:"USER" description:"smtp user name"`
+			Port         int           `long:"port" env:"PORT" description:"[deprecated, use --smtp.port] SMTP password"`
-			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"smtp password"`
+			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"[deprecated, use --smtp.password] SMTP port"`
-			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"smtp timeout"`
+			SMTPUserName string        `long:"user" env:"USER" description:"[deprecated, use --smtp.username] enable TLS"`
+			TLS          bool          `long:"tls" env:"TLS" description:"[deprecated, use --smtp.tls] SMTP TCP connection timeout"`
+			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"[deprecated, use --smtp.timeout] SMTP TCP connection timeout"`
+			MsgTemplate  string        `long:"template" env:"TEMPLATE" description:"message template file"`
 		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
 }
 
+// ImageProxyGroup defines options group for image proxy
+type ImageProxyGroup struct {
+	HTTP2HTTPS    bool `long:"http2https" env:"HTTP2HTTPS" description:"enable HTTP->HTTPS proxy"`
+	CacheExternal bool `long:"cache-external" env:"CACHE_EXTERNAL" description:"enable caching for external images"`
+}
+
 // AuthGroup defines options group for auth params
 type AuthGroup struct {
 	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
@@ -103,17 +119,17 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"` // nolint
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
-	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"TPC"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -123,25 +139,26 @@ type ImageGroup struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
 	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
-	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
-	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
 }
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"uri" default:"fs"` //nolint
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+	URI    string `long:"uri" env:"URI" default:"./var/avatars" description:"avatar's store URI"`
+	RszLmt int    `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
 }
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` // nolint
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -151,7 +168,7 @@ type CacheGroup struct {
 
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"` //nolint
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
@@ -159,21 +176,35 @@ type AdminGroup struct {
 	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
+// SmtpGroup defines options for SMTP server connection, used in auth and notify modules
+type SmtpGroup struct {
+	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
+	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
+	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Password string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	TLS      bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	TimeOut  time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
+}
+
 // NotifyGroup defines options for notification
 type NotifyGroup struct {
-	Type      string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" default:"none"`
+	Type      []string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" choice:"email" default:"none" env-delim:","` //nolint
-	QueueSize int    `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
+	QueueSize int      `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
 	Telegram  struct {
 		Token   string        `long:"token" env:"TOKEN" description:"telegram token"`
 		Channel string        `long:"chan" env:"CHAN" description:"telegram channel"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"telegram timeout"`
 		API     string        `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
 	} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
+	Email struct {
+		From                string `long:"fromAddress" env:"FROM" description:"from email address"`
+		VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
+	} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 }
 
 // SSLGroup defines options group for server ssl params
 type SSLGroup struct {
-	Type         string `long:"type" env:"TYPE" description:"ssl (auto)support" choice:"none" choice:"static" choice:"auto" default:"none"`
+	Type         string `long:"type" env:"TYPE" description:"ssl (auto) support" choice:"none" choice:"static" choice:"auto" default:"none"` //nolint
 	Port         int    `long:"port" env:"PORT" description:"port number for https server" default:"8443"`
 	Cert         string `long:"cert" env:"CERT" description:"path to cert.pem file"`
 	Key          string `long:"key" env:"KEY" description:"path to key.pem file"`
@@ -196,6 +227,12 @@ type RPCGroup struct {
 	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
 }
 
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req cache.FlusherRequest)                                        // evict matched records
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -237,6 +274,40 @@ func (s *ServerCommand) Execute(args []string) error {
 	return nil
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated returns their list
+func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
+	// 1.5.0
+	if s.Auth.Email.Host != "" && s.SMTP.Host == "" {
+		s.SMTP.Host = s.Auth.Email.Host
+		result = append(result, DeprecatedFlag{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.Port != 0 && s.SMTP.Port == 0 {
+		s.SMTP.Port = s.Auth.Email.Port
+		result = append(result, DeprecatedFlag{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TLS && !s.SMTP.TLS {
+		s.SMTP.TLS = s.Auth.Email.TLS
+		result = append(result, DeprecatedFlag{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPUserName != "" && s.SMTP.Username == "" {
+		s.SMTP.Username = s.Auth.Email.SMTPUserName
+		result = append(result, DeprecatedFlag{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPPassword != "" && s.SMTP.Password == "" {
+		s.SMTP.Password = s.Auth.Email.SMTPPassword
+		result = append(result, DeprecatedFlag{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TimeOut != 10*time.Second && s.SMTP.TimeOut == 10*time.Second {
+		s.SMTP.TimeOut = s.Auth.Email.TimeOut
+		result = append(result, DeprecatedFlag{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"})
+	}
+	if s.LegacyImageProxy && !s.ImageProxy.HTTP2HTTPS {
+		s.ImageProxy.HTTP2HTTPS = s.LegacyImageProxy
+		result = append(result, DeprecatedFlag{Old: "img-proxy", New: "image-proxy.http2https", RemoveVersion: "1.7.0"})
+	}
+	return result
+}
+
 // newServerApp prepares application and return it with all active parts
 // doesn't start anything
 func (s *ServerCommand) newServerApp() (*serverApp, error) {
@@ -277,6 +348,8 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
+	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
+	dataService.RestrictSameIPVotes.Duration = s.DurationVoteIP
 
 	loadingCache, err := s.makeCache()
 	if err != nil {
@@ -297,16 +370,33 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
+		UrlMapperMaker:    migrator.NewUrlMapper,
 		KeyStore:          adminStore,
 	}
 
-	notifyService, err := s.makeNotify(dataService)
+	var emailNotifications bool
+	notifyService, err := s.makeNotify(dataService, authenticator)
+
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "email":
+			emailNotifications = true
+		}
+	}
+
 	if err != nil {
 		log.Printf("[WARN] failed to make notify service, %s", err)
 		notifyService = notify.NopService // disable notifier
+		emailNotifications = false        // email notifications are not available in this case
 	}
 
-	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
+	imgProxy := &proxy.Image{
+		HTTP2HTTPS:    s.ImageProxy.HTTP2HTTPS,
+		CacheExternal: s.ImageProxy.CacheExternal,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     s.RemarkURL,
+		ImageService:  imageService,
+	}
 	emojiFmt := store.CommentConverterFunc(func(text string) string { return text })
 	if s.EnableEmoji {
 		emojiFmt = func(text string) string { return emoji.Sprint(text) }
@@ -339,7 +429,10 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 			Refresh:   s.Stream.RefreshInterval,
 			MaxActive: int32(s.Stream.MaxActive),
 		},
-		EmojiEnabled: s.EnableEmoji,
+		EmailNotifications: emailNotifications,
+		EmojiEnabled:       s.EnableEmoji,
+		AnonVote:           s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:         s.SimpleView,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -438,7 +531,7 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
 	case "rpc":
-		r := &engine.RPC{Client: rpc.Client{
+		r := &engine.RPC{Client: jrpc.Client{
 			API:        s.Store.RPC.API,
 			Client:     http.Client{Timeout: s.Store.RPC.TimeOut},
 			AuthUser:   s.Store.RPC.AuthUser,
@@ -465,12 +558,30 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
+	case "uri":
+		return avatar.NewStore(s.Avatar.URI)
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 	switch s.Image.Type {
+	case "bolt":
+		boltImageStore, err := image.NewBoltStorage(
+			s.Image.Bolt.File,
+			s.Image.MaxSize,
+			s.Image.ResizeHeight,
+			s.Image.ResizeWidth,
+			bolt.Options{},
+		)
+		if err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store:    boltImageStore,
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
+		}, nil
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
@@ -501,9 +612,9 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 				s.Admin.Shared.Email = "admin@" + u.Host
 			}
 		}
-		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
+		return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
 	case "rpc":
-		r := &admin.RPC{Client: rpc.Client{
+		r := &admin.RPC{Client: jrpc.Client{
 			API:        s.Admin.RPC.API,
 			Client:     http.Client{Timeout: s.Admin.RPC.TimeOut},
 			AuthUser:   s.Admin.RPC.AuthUser,
@@ -515,14 +626,18 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	}
 }
 
-func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
+func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
 	switch s.Cache.Type {
 	case "mem":
-		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
+		if err != nil {
+			return nil, errors.Wrap(err, "cache backend initialization")
+		}
+		return cache.NewScache(backend), nil
 	case "none":
-		return &cache.Nop{}, nil
+		return cache.NewScache(&cache.Nop{}), nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
@@ -537,10 +652,10 @@ var msgTemplate = `
 <body>
 <div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
 	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
-	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation&nbsp;for <b>{{.User}}</b> on&nbsp;site&nbsp;<b>{{.Site}}</b></p>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
 	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
 		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
-		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and&nbsp;paste this text into “token” field on&nbsp;comments page</i></p>
+		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and paste this text into “token” field on comments page</i></p>
 		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
 	</div>
 	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
@@ -568,6 +683,11 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
 		providers++
 	}
+	if s.Auth.Twitter.CID != "" && s.Auth.Twitter.CSEC != "" {
+		authenticator.AddProvider("twitter", s.Auth.Twitter.CID, s.Auth.Twitter.CSEC)
+		providers++
+	}
+
 	if s.Auth.Dev {
 		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
@@ -576,18 +696,18 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
 	if s.Auth.Email.Enable {
 		params := sender.EmailParams{
-			Host:         s.Auth.Email.Host,
+			Host:         s.SMTP.Host,
-			Port:         s.Auth.Email.Port,
+			Port:         s.SMTP.Port,
+			SMTPUserName: s.SMTP.Username,
+			SMTPPassword: s.SMTP.Password,
+			TimeOut:      s.SMTP.TimeOut,
+			TLS:          s.SMTP.TLS,
 			From:         s.Auth.Email.From,
 			Subject:      s.Auth.Email.Subject,
 			ContentType:  s.Auth.Email.ContentType,
-			TLS:          s.Auth.Email.TLS,
-			SMTPUserName: s.Auth.Email.SMTPUserName,
-			SMTPPassword: s.Auth.Email.SMTPPassword,
-			TimeOut:      s.Auth.Email.TimeOut,
 		}
 		sndr := sender.NewEmailClient(params, log.Default())
-		authenticator.AddVerifProvider("email", msgTemplate, sndr)
+		authenticator.AddVerifProvider("email", s.loadEmailTemplate(), sndr)
 	}
 
 	if s.Auth.Anonymous {
@@ -613,20 +733,83 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 	}
 }
 
-func (s *ServerCommand) makeNotify(dataStore *service.DataStore) (*notify.Service, error) {
+// loadEmailTemplate trying to get template from opts MsgTemplate and default to embedded
-	log.Printf("[INFO] make notify, type=%s", s.Notify.Type)
+// if not defined or failed to load
-	switch s.Notify.Type {
+func (s *ServerCommand) loadEmailTemplate() string {
-	case "telegram":
+	tmpl := msgTemplate
-		tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
+	if s.Auth.Email.MsgTemplate != "" {
-			s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
+		log.Printf("[DEBUG] load email template from %s", s.Auth.Email.MsgTemplate)
-		if err != nil {
+		b, err := ioutil.ReadFile(s.Auth.Email.MsgTemplate)
-			return nil, errors.Wrap(err, "failed to create telegram notification destination")
+		if err == nil {
+			tmpl = string(b)
+		} else {
+			log.Printf("[WARN] failed to load email template from %s, %v", s.Auth.Email.MsgTemplate, err)
 		}
-		return notify.NewService(dataStore, s.Notify.QueueSize, tg), nil
-	case "none":
-		return notify.NopService, nil
 	}
-	return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+	return tmpl
+}
+
+func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *auth.Service) (*notify.Service, error) {
+	var notifyService *notify.Service
+	var destinations []notify.Destination
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "telegram":
+			tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
+				s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create telegram notification destination")
+			}
+			destinations = append(destinations, tg)
+		case "email":
+			emailParams := notify.EmailParams{
+				From:                s.Notify.Email.From,
+				VerificationSubject: s.Notify.Email.VerificationSubject,
+				UnsubscribeURL:      s.RemarkURL + "/email/unsubscribe.html",
+				// TODO: uncomment after #560 frontend part is ready and URL is known
+				//SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
+				TokenGenFn: func(userID, email, site string) (string, error) {
+					claims := token.Claims{
+						Handshake: &token.Handshake{ID: userID + "::" + email},
+						StandardClaims: jwt.StandardClaims{
+							Audience:  site,
+							ExpiresAt: time.Now().Add(100 * 365 * 24 * time.Hour).Unix(),
+							NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+							Issuer:    "remark42",
+						},
+					}
+					tkn, err := authenticator.TokenService().Token(claims)
+					if err != nil {
+						return "", errors.Wrapf(err, "failed to make unsubscription token")
+					}
+					return tkn, nil
+				},
+			}
+			smtpParams := notify.SmtpParams{
+				Host:     s.SMTP.Host,
+				Port:     s.SMTP.Port,
+				TLS:      s.SMTP.TLS,
+				Username: s.SMTP.Username,
+				Password: s.SMTP.Password,
+				TimeOut:  s.SMTP.TimeOut,
+			}
+			emailService, err := notify.NewEmail(emailParams, smtpParams)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create email notification destination")
+			}
+			destinations = append(destinations, emailService)
+		case "none":
+			notifyService = notify.NopService
+		default:
+			return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+		}
+	}
+
+	if len(destinations) != 0 {
+		log.Printf("[INFO] make notify, types=%s", s.Notify.Type)
+		notifyService = notify.NewService(dataStore, s.Notify.QueueSize, destinations...)
+	}
+	return notifyService, nil
 }
 
 func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
@@ -675,6 +858,11 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			c.User.SetAdmin(ds.IsAdmin(c.Audience, c.User.ID))
 			c.User.SetBoolAttr("blocked", ds.IsBlocked(c.Audience, c.User.ID))
+			var err error
+			c.User.Email, err = ds.GetUserEmail(c.Audience, c.User.ID)
+			if err != nil {
+				log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
+			}
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
@@ -682,6 +870,9 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			if claims.User == nil {
 				return false
 			}
+			if claims.User.Audience == "" { // reject empty aud, made with old (pre 0.8.x) version of auth package
+				return false
+			}
 			return !claims.User.BoolAttr("blocked")
 		}),
 		JWTQuery:          "jwt", // change default from "token" as it used for deleteme

backend/app/cmd/server_test.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -15,7 +17,6 @@ import (
 
 	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
-	log "github.com/go-pkgz/lgr"
 	"github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
@@ -23,32 +24,33 @@ import (
 )
 
 func TestServerApp(t *testing.T) {
-	port := rand.Intn(40000) + 10000
+	port := chooseRandomUnusedPort()
-	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
 		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(250 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
+	client := http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 	body, _ = ioutil.ReadAll(resp.Body)
 	t.Log(string(body))
@@ -57,12 +59,13 @@ func TestServerApp(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
-	port := rand.Intn(40000) + 10000
+	port := chooseRandomUnusedPort()
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
 		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
@@ -70,90 +73,89 @@ func TestServerApp_DevMode(t *testing.T) {
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_AnonMode(t *testing.T) {
-	port := rand.Intn(40000) + 10000
+	port := chooseRandomUnusedPort()
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
 		o.Port = port
 		o.Auth.Anonymous = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
-	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[5].Name(), "anon auth provider")
 
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
 	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	// try to login with bad name
 	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
 	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://localhost:18443", SharedSecret: "123456"})
+	sslPort := chooseRandomUnusedPort()
+	opts.SetCommon(CommonOpts{RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort), SharedSecret: "123456"})
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=" + strconv.Itoa(port), "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
 		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
 		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
-		"--ssl.port=18443", "--image.fs.path=/tmp"})
+		"--ssl.port=" + strconv.Itoa(sslPort), "--image.fs.path=/tmp"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	// create app
 	app, err := opts.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(1 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -168,21 +170,22 @@ func TestServerApp_WithSSL(t *testing.T) {
 	}
 
 	// check http to https redirect response
-	resp, err := client.Get("http://localhost:18080/blah?param=1")
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:18443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	// check https server
-	resp, err = client.Get("https://localhost:18443/ping")
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
@@ -193,35 +196,32 @@ func TestServerApp_WithRemote(t *testing.T) {
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
+	port := chooseRandomUnusedPort()
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
 		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
-		"--port=12345", "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
 	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
 
 	// create app
 	app, err := opts.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
@@ -233,7 +233,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 	// RO bolt location
 	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -243,7 +243,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -253,7 +253,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
@@ -262,7 +262,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
-	assert.NotNil(t, err, "blah is invalid type")
+	assert.Error(t, err, "blah is invalid type")
 
 	opts.Store.Type = "blah"
 	_, err = opts.newServerApp()
@@ -271,38 +271,86 @@ func TestServerApp_Failed(t *testing.T) {
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
-		o.Port = rand.Intn(40000) + 10000
+		o.Port = chooseRandomUnusedPort()
 		return o
 	})
+	time.AfterFunc(100*time.Millisecond, func() {
+		cancel()
+	})
 	st := time.Now()
 	err := app.run(ctx)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100msec")
 	app.Wait()
 }
 
 func TestServerApp_MainSignal(t *testing.T) {
 
+	done := make(chan struct{})
 	go func() {
+		<-done
 		time.Sleep(250 * time.Millisecond)
 		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}()
-	st := time.Now()
 
 	s := ServerCommand{}
 	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	p := flags.NewParser(&s, flags.Default)
+	port := chooseRandomUnusedPort()
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=" + strconv.Itoa(port), "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	st := time.Now()
+	close(done)
 	err = s.Execute(args)
-	assert.NoError(t, err, "execute failed")
+	assert.NoError(t, err, "execute should be without errors")
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.True(t, time.Since(st).Seconds() < 5, "should take under five sec", time.Since(st).Seconds())
+}
+
+func TestServerApp_DeprecatedArgs(t *testing.T) {
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{
+		"test",
+		"--auth.email.host=smtp.example.org",
+		"--auth.email.port=666",
+		"--auth.email.tls",
+		"--auth.email.user=test_user",
+		"--auth.email.passwd=test_password",
+		"--auth.email.timeout=15s",
+	}
+	assert.Empty(t, s.SMTP.Host)
+	assert.Empty(t, s.SMTP.Port)
+	assert.Empty(t, s.SMTP.TLS)
+	assert.Empty(t, s.SMTP.Username)
+	assert.Empty(t, s.SMTP.Password)
+	assert.Empty(t, s.SMTP.TimeOut)
+	_, err := p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlags := s.HandleDeprecatedFlags()
+	assert.ElementsMatch(t,
+		[]DeprecatedFlag{
+			{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"},
+		},
+		deprecatedFlags)
+	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
+	assert.Equal(t, 666, s.SMTP.Port)
+	assert.Equal(t, true, s.SMTP.TLS)
+	assert.Equal(t, "test_user", s.SMTP.Username)
+	assert.Equal(t, "test_password", s.SMTP.Password)
+	assert.Equal(t, 15*time.Second, s.SMTP.TimeOut)
 }
 
 func Test_ACMEEmail(t *testing.T) {
@@ -311,9 +359,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	args := []string{"--ssl.type=auto"}
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err := cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -321,9 +369,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--ssl.acme-email=adminname@adminhost.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "adminname@adminhost.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -331,9 +379,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared", "--admin.shared.email=superadmin@admin.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "superadmin@admin.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -341,21 +389,21 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	port := rand.Intn(40000) + 10000
+	port := chooseRandomUnusedPort()
-	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
 		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// make a token for user dev
 	tkService := app.restSrv.Authenticator.TokenService()
@@ -378,7 +426,7 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 1 * time.Second}
+	client := http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	require.NoError(t, err)
@@ -388,42 +436,112 @@ func TestServerAuthHooks(t *testing.T) {
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
+	// add comment with no-aud claim
+	claimsNoAud := claims
+	claimsNoAud.Audience = ""
+	tkNoAud, err := tkService.Token(claimsNoAud)
+	require.NoError(t, err)
+	t.Logf("no-aud claims: %s", tkNoAud)
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/", 
+"site": "remark"}}`))
+	require.NoError(t, err)
+	req.Header.Set("X-JWT", tkNoAud)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
+
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut,
+	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
-	assert.Nil(t, e)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, e = client.Do(req)
+	resp, err = client.Do(req)
-	require.Nil(t, e)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Log(string(b))
 
-	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
-
 	// try add a comment with blocked user
 	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
+		"blocked user can't post, \n"+tk+"\n"+string(body))
 
+	cancel()
 	app.Wait()
 }
 
-func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
+func TestServer_loadEmailTemplate(t *testing.T) {
+	cmd := ServerCommand{}
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
+	r := cmd.loadEmailTemplate()
+	assert.Equal(t, "The token is {{.Token}}", r)
+
+	cmd.Auth.Email.MsgTemplate = ""
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+
+	cmd.Auth.Email.MsgTemplate = "bad-file"
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}
+
+func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
 	cmd := ServerCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "secret"})
 
 	// prepare options
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
@@ -431,23 +549,27 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
 	cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
+	cmd.Auth.Email.Enable = true
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
 	cmd.BackupLocation = "/tmp"
-	cmd.Notify.Type = "telegram"
+	cmd.Notify.Type = []string{"email"}
-	cmd.Notify.Telegram.API = "http://127.0.0.1:12340/"
+	cmd.Notify.Email.From = "from@example.org"
-	cmd.Notify.Telegram.Token = "blah"
+	cmd.Notify.Email.VerificationSubject = "test verification email subject"
+	cmd.SMTP.Host = "127.0.0.1"
+	cmd.SMTP.Port = 25
+	cmd.SMTP.Username = "test_user"
+	cmd.SMTP.Password = "test_password"
+	cmd.SMTP.TimeOut = time.Second
+	cmd.UpdateLimit = 10
 	cmd = fn(cmd)
 
 	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
 
 	// create app
 	app, err := cmd.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	time.AfterFunc(duration, func() {
-		log.Print("[TEST] terminate app")
-		cancel()
-	})
 	rand.Seed(time.Now().UnixNano())
-	return app, ctx
+	return app, ctx, cancel
 }

backend/app/cmd/testdata/email.tmpl

@@ -0,0 +1 @@
+The token is {{.Token}}

backend/app/cmd/testdata/remap_urls.txt

@@ -0,0 +1,2 @@
+http://oldsite.com* https://newsite.com*
+http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1

backend/app/main.go

@@ -21,6 +21,7 @@ type Opts struct {
 	RestoreCmd cmd.RestoreCommand `command:"restore"`
 	AvatarCmd  cmd.AvatarCommand  `command:"avatar"`
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
 	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
@@ -44,6 +45,10 @@ func main() {
 			SharedSecret: opts.SharedSecret,
 			Revision:     revision,
 		})
+		for _, entry := range c.HandleDeprecatedFlags() {
+			log.Printf("[WARN] --%s is deprecated and will be removed in v%s, please use --%s instead",
+				entry.Old, entry.RemoveVersion, entry.New)
+		}
 		err := c.Execute(args)
 		if err != nil {
 			log.Printf("[ERROR] failed with %+v", err)

backend/app/main_test.go

@@ -1,19 +1,19 @@
 package main
 
 import (
-	"context"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/repeater"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,45 +24,37 @@ func Test_Main(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
+	port := chooseRandomUnusedPort()
 	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
-		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+		"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
+	done := make(chan struct{})
 	go func() {
-		time.Sleep(5000 * time.Millisecond)
+		<-done
-		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, e)
+		require.NoError(t, err)
 	}()
 
-	wg := sync.WaitGroup{}
+	finished := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
+		close(finished)
-		wg.Done()
 	}()
 
-	var passed bool
+	// defer cleanup because require check below can fail
-	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
+	defer func() {
-		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		close(done)
-		if e != nil {
+		<-finished
-			t.Logf("%+v", e)
+	}()
-			return e
-		}
-		require.Nil(t, e)
-		defer resp.Body.Close()
-		assert.Equal(t, 200, resp.StatusCode)
-		body, e := ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
-		assert.Equal(t, "pong", string(body))
-		passed = true
-		return nil
-	})
 
+	waitForHTTPServerStart(port)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, true, passed, "at least on ping passed")
+	assert.Equal(t, "pong", string(body))
-
-	wg.Wait()
 }
 
 func TestGetDump(t *testing.T) {
@@ -72,3 +64,26 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
 	log.Printf("\n dump: %s", dump)
 }
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 10 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 100; i++ {
+		time.Sleep(time.Millisecond * 100)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}

backend/app/migrator/backup_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBackup_RemoveOldBackupFiles(t *testing.T) {
@@ -21,17 +22,17 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 	}
 	fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
 	err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
 	bk.removeOldBackupFiles()
 	ff, err := ioutil.ReadDir(loc)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
+	require.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
 	assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
 	assert.Equal(t, "backup-site1-20171209.gz", ff[1].Name())
 	assert.Equal(t, "backup-site1-20171210.gz", ff[2].Name())

backend/app/migrator/disqus_test.go

@@ -19,16 +19,17 @@ import (
 func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
+	require.NoError(t, err, "create store")
-	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	d := Disqus{DataStore: &dataStore}
 	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
 	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 4, len(last), "4 comments imported")
+	require.Equal(t, 4, len(last), "4 comments imported")
 
 	c := last[len(last)-1] // last reverses, get first one
 	assert.True(t, strings.HasPrefix(c.Text, "<p>The quick brown fox"))
@@ -40,11 +41,11 @@ func TestDisqus_Import(t *testing.T) {
 	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
 
 	posts, err := dataStore.List("test", 0, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts), "2 posts")
 
 	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, count)
 }
 
@@ -56,7 +57,7 @@ func TestDisqus_Convert(t *testing.T) {
 	for comment := range ch {
 		res = append(res, comment)
 	}
-	assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
+	require.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
 
 	exp0 := store.Comment{
 		ID: "299619020",

backend/app/migrator/mapper.go

@@ -0,0 +1,74 @@
+package migrator
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+// UrlMapper implements Mapper interface
+type UrlMapper struct {
+	rules map[string]string
+}
+
+// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
+// if given rules are valid.
+func NewUrlMapper(reader io.Reader) (Mapper, error) {
+	u := &UrlMapper{}
+	if err := u.loadRules(reader); err != nil {
+		return u, err
+	}
+	return u, nil
+}
+
+// loadRules loads url-mapping rules from reader to mapper.
+// Rules must be a text consists of rows separated by \n.
+// Each row holds from-url and to-url separated by space.
+// If urls end with asterisk (*) it means try to match by prefix.
+// Example:
+// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
+// https://www.myblog.com/* https://myblog.com/*
+func (u *UrlMapper) loadRules(reader io.Reader) error {
+	data, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	rulesText := strings.TrimSpace(string(data))
+
+	u.rules = make(map[string]string)
+
+	for _, row := range strings.Split(rulesText, "\n") {
+		row = strings.TrimSpace(row)
+		urls := strings.Split(row, " ")
+		if len(urls) != 2 {
+			return errors.New("bad row " + row)
+		}
+
+		from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
+		u.rules[from] = to
+	}
+	return nil
+}
+
+// URL maps given url to another url according loaded url-rules.
+// If not matched returns given url.
+func (u *UrlMapper) URL(url string) string {
+	if newUrl, ok := u.rules[url]; ok {
+		return newUrl
+	}
+	// try to match by prefix
+	for oldUrl, newUrl := range u.rules {
+		if !strings.HasSuffix(oldUrl, "*") {
+			continue
+		}
+		oldUrl = strings.TrimSuffix(oldUrl, "*")
+		newUrl = strings.TrimSuffix(newUrl, "*")
+		if strings.HasPrefix(url, oldUrl) {
+			return newUrl + strings.TrimPrefix(url, oldUrl)
+		}
+	}
+	// search failed, return given url
+	return url
+}

backend/app/migrator/mapper_test.go

@@ -0,0 +1,90 @@
+package migrator
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUrlMapper_URL(t *testing.T) {
+	// want remap urls from https://radio-t.com to https://www.radio-t.com
+	// also map individual urls
+	rules := strings.NewReader(`
+https://radio-t.com* https://www.radio-t.com*
+https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
+https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
+`)
+
+	mapper, err := NewUrlMapper(rules)
+	assert.NoError(t, err)
+
+	// if url not matched mapper should return given url
+	assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
+	assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
+	// check strict matching
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
+	// check pattern matching (by prefix)
+	assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
+
+	// want remap from http to https
+	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
+	mapper, err = NewUrlMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
+
+	// want remap from http to https by pattern
+	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
+	mapper, err = NewUrlMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+}
+
+func TestUrlMapper_New(t *testing.T) {
+	cases := []struct {
+		rules       string
+		expectError bool
+	}{
+		// bad input, expect error
+		{
+			rules:       "https://radio-t.com ",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com\n https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com   \n https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+
+		// valid input, no error
+		{
+			rules: "https://radio-t.com* https://www.radio-t.com*",
+		},
+		{
+			rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
+		},
+		{
+			rules: "https://any.com/p/111 https://any.com/p/222   \n https://any.com/p/333 https://any.com/p/222   \n",
+		},
+	}
+	for _, c := range cases {
+		_, err := NewUrlMapper(strings.NewReader(c.rules))
+		if c.expectError {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}

backend/app/migrator/migrator.go

@@ -24,6 +24,16 @@ type Exporter interface {
 	Export(w io.Writer, siteID string) (int, error)
 }
 
+// Mapper defines interface to convert data in import procedure
+type Mapper interface {
+	URL(url string) string
+}
+
+// MapperMaker defines function that reads rules from reader and
+// returns new Mapper with loaded rules. If rules are not valid
+// it returns error.
+type MapperMaker func(reader io.Reader) (Mapper, error)
+
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)

backend/app/migrator/migrator_test.go

@@ -23,22 +23,23 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
+	require.NoError(t, err, "create store")
-	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
 		SiteID:    "test",
 		Provider:  "disqus",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
 	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
 
@@ -49,22 +50,23 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
+	require.NoError(t, err, "create store")
-	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
 		SiteID:    "test",
 		Provider:  "wordpress",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
 	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
@@ -78,11 +80,12 @@ func TestMigrator_ImportNative(t *testing.T) {
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
 	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
+	require.NoError(t, err, "create store")
-	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -90,19 +93,20 @@ func TestMigrator_ImportNative(t *testing.T) {
 		SiteID:    "radio-t",
 		Provider:  "native",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
 
 func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
+	require.NoError(t, err, "create store")
 	dataStore := &service.DataStore{Engine: b}
+	defer dataStore.Close()
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
 	return nil
 }
 
+// WithMapper wraps reader with url-mapper.
+func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			log.Printf("[DEBUG] finish write to pipe with %+v", err)
+			if e := w.Close(); e != nil {
+				log.Printf("[WARN] failed close pipe writer with %+v", e)
+			}
+		}()
+
+		// decode from reader and encode to pipe writer
+		dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
+
+		m := meta{}
+		if err = dec.Decode(&m); err != nil {
+			return
+		}
+		for i := range m.Posts {
+			m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
+		}
+		if err = enc.Encode(m); err != nil {
+			return
+		}
+
+		for {
+			comment := store.Comment{}
+			if err = dec.Decode(&comment); err != nil {
+				return
+			}
+			comment.Locator.URL = mapper.URL(comment.Locator.URL)
+			if err = enc.Encode(comment); err != nil {
+				return
+			}
+		}
+	}()
+
+	return r
+}
+
 // Import comments from json strings produced by Remark.Export
 func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
-
 	m := meta{}
 	dec := json.NewDecoder(reader)
 	if err = dec.Decode(&m); err != nil {

backend/app/migrator/native_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
@@ -20,11 +21,9 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
-
 func TestNative_Export(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
-	b := prep(t) // write 2 comments
+	defer teardown()
 	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
 	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
@@ -32,7 +31,7 @@ func TestNative_Export(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
@@ -48,7 +47,7 @@ func TestNative_Export(t *testing.T) {
 
 	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(m.Users))
+	require.Equal(t, 2, len(m.Users))
 	assert.Equal(t, "user1", m.Users[0].ID)
 	assert.Equal(t, false, m.Users[0].Blocked.Status)
 	assert.Equal(t, true, m.Users[0].Verified)
@@ -56,7 +55,7 @@ func TestNative_Export(t *testing.T) {
 	assert.Equal(t, true, m.Users[1].Blocked.Status)
 	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(m.Posts))
+	require.Equal(t, 1, len(m.Posts))
 	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
 	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
@@ -70,22 +69,22 @@ func TestNative_Export(t *testing.T) {
 }
 
 func TestNative_Import(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
-	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
 	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments))
+	require.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
 	assert.Equal(t, "1234", comments[0].ParentID)
 	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
@@ -101,13 +100,54 @@ func TestNative_Import(t *testing.T) {
 	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
 }
 
+func TestNative_ImportWithMapper(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
+	// want to remap comments to https://rdt.c
+	rules := `https://radio-t.com* https://rdt.c*`
+	mapper, err := NewUrlMapper(strings.NewReader(rules))
+	assert.NoError(t, err)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	mappedReader := WithMapper(strings.NewReader(inp), mapper)
+
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
+	size, err := r.Import(mappedReader, "radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
 func TestNative_ImportWrongVersion(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
 	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
-	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
 	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
@@ -115,37 +155,38 @@ func TestNative_ImportWrongVersion(t *testing.T) {
 
 }
 func TestNative_ImportManyWithError(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
 
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
-	for i := 0; i < 1200; i++ {
+	for i := 0; i < 100; i++ {
 		buf.WriteString(fmt.Sprintf(goodRec, i))
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")
 
-	b := prep(t) // write some recs
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
-	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
 	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
+	assert.Equal(t, 100, n)
 	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 1200, len(comments))
+	assert.Equal(t, 100, len(comments))
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
+func prep(t *testing.T) (*service.DataStore, func()) {
-	os.Remove(testDb)
+
+	testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
@@ -155,7 +196,7 @@ func prep(t *testing.T) *service.DataStore {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
@@ -163,7 +204,10 @@ func prep(t *testing.T) *service.DataStore {
 		User:    store.User{ID: "user2", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	return b
+	return b, func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
 }

backend/app/migrator/wordpress_test.go

@@ -8,6 +8,7 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -19,17 +20,18 @@ func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
 	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
-	assert.Nil(t, err, "create store")
+	assert.NoError(t, err, "create store")
 
-	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
 	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
+	require.Equal(t, 3, len(last), "3 comments imported")
 
 	c := last[0]
 	assert.Equal(t, "14", c.ID)
@@ -42,14 +44,14 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
 
 	posts, err := dataStore.List(siteID, 0, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 1, len(posts))
+	require.Equal(t, 1, len(posts))
 
 	p := posts[0]
 	assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
 
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
 }
 
@@ -61,7 +63,7 @@ func TestWordPress_Convert(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
+	require.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
 
 	exp1 := store.Comment{
 		ID: "13",
@@ -88,7 +90,7 @@ func TestWP_Convert_MD(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported")
+	require.Equal(t, 3, len(comments), "3 comments exported")
 
 	assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
 

backend/app/notify/email.go

@@ -0,0 +1,456 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"mime/quotedprintable"
+	"net"
+	"net/smtp"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
+	"github.com/pkg/errors"
+)
+
+// EmailParams contain settings for email notifications
+type EmailParams struct {
+	From                 string // from email address
+	MsgTemplate          string // request message template
+	VerificationSubject  string // verification message subject
+	VerificationTemplate string // verification message template
+	SubscribeURL         string // full subscribe handler URL
+	UnsubscribeURL       string // full unsubscribe handler URL
+
+	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
+}
+
+// SmtpParams contain settings for smtp server connection
+type SmtpParams struct {
+	Host     string        // SMTP host
+	Port     int           // SMTP port
+	TLS      bool          // TLS auth
+	Username string        // user name
+	Password string        // password
+	TimeOut  time.Duration // TCP connection timeout
+}
+
+// Email implements notify.Destination for email
+type Email struct {
+	EmailParams
+	SmtpParams
+
+	smtp       smtpClientCreator
+	msgTmpl    *template.Template // parsed request message template
+	verifyTmpl *template.Template // parsed verification message template
+}
+
+// default email client implementation
+type emailClient struct{ smtpClientCreator }
+
+// smtpClient interface defines subset of net/smtp used by email client
+type smtpClient interface {
+	Mail(string) error
+	Auth(smtp.Auth) error
+	Rcpt(string) error
+	Data() (io.WriteCloser, error)
+	Quit() error
+	Close() error
+}
+
+// smtpClientCreator interface defines function for creating new smtpClients
+type smtpClientCreator interface {
+	Create(SmtpParams) (smtpClient, error)
+}
+
+type emailMessage struct {
+	from    string
+	to      string
+	message string
+}
+
+// msgTmplData store data for message from request template execution
+type msgTmplData struct {
+	UserName          string
+	UserPicture       string
+	CommentText       string
+	CommentLink       string
+	CommentDate       time.Time
+	ParentUserName    string
+	ParentUserPicture string
+	ParentCommentText string
+	ParentCommentLink string
+	ParentCommentDate time.Time
+	PostTitle         string
+	Email             string
+	UnsubscribeLink   string
+}
+
+// verifyTmplData store data for verification message template execution
+type verifyTmplData struct {
+	User         string
+	Token        string
+	Email        string
+	Site         string
+	SubscribeURL string
+}
+
+const (
+	defaultVerificationSubject = "Email verification"
+	defaultEmailTimeout        = 10 * time.Second
+	defaultEmailTemplate       = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+	<style type="text/css">
+		img {
+			max-width: 100%;
+			max-height: 250px;
+			margin: 5px 0;
+			display: block;
+			color: #000;
+		}
+		a {
+			text-decoration: none;
+			color: #0aa;
+		}
+		p {
+			margin: 0 0 12px;
+		}
+	</style>
+</head>
+<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+<body>
+	<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
+		<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
+		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+		<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
+			<div>
+				<div style="margin-bottom: 12px; line-height: 24px;">
+					<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+					<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
+					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
+					<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
+				</div>
+				<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
+					{{.ParentCommentText}}
+				</div>
+			</div>
+			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px; line-height: 24px;">
+				<div style="margin-bottom: 8px;">
+					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+					<div style="float: left; font-size: 14px; font-weight: bold; color: #777">
+						{{.UserName}}
+					</div>
+					<div style="color: #999; font-size: 14px; margin: 0 8px; float: left;">
+						{{.CommentDate.Format "02.01.2006 at 15:04"}}
+					</div>
+					<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
+				</div>
+				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">
+					{{.CommentText}}
+				</div>
+			</div>
+		</div>
+		<div style="text-align: center; font-size: 14px; margin-top: 32px;">
+			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a> for {{.ParentUserName}}</i>
+			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;">
+				<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
+				<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
+				<div style="opacity: 0;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
+			</div>
+		</div>
+	</div>
+</body>
+</html>
+`
+	defaultEmailVerificationTemplate = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+	<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+	<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
+		<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+		<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
+		{{if .SubscribeURL}}
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
+		{{ end }}
+		<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+			<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
+			<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
+			<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+		</div>
+		<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
+	</div>
+</body>
+</html>
+`
+)
+
+// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
+func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
+	// set up Email emailParams
+	res := Email{EmailParams: emailParams}
+	if res.MsgTemplate == "" {
+		res.MsgTemplate = defaultEmailTemplate
+	}
+	if res.VerificationTemplate == "" {
+		res.VerificationTemplate = defaultEmailVerificationTemplate
+	}
+	if res.VerificationSubject == "" {
+		res.VerificationSubject = defaultVerificationSubject
+	}
+
+	// set up SMTP emailParams
+	res.smtp = &emailClient{}
+	res.SmtpParams = smtpParams
+	if res.TimeOut <= 0 {
+		res.TimeOut = defaultEmailTimeout
+	}
+
+	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
+		res.Host, res.Username, res.TimeOut)
+
+	// initialise templates
+	var err error
+	if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse message template")
+	}
+	if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse verification template")
+	}
+	return &res, err
+}
+
+// Send email about reply to Request.Email if it's set, otherwise do nothing and return nil, thread safe
+// do not returns sending error, only following:
+// 1. (likely impossible) template execution error from email message creation from Request
+// 2. message dropped without sending in case of closed ctx
+func (e *Email) Send(ctx context.Context, req Request) (err error) {
+	if req.Email == "" {
+		// this means we can't send this request via Email
+		return nil
+	}
+	select {
+	case <-ctx.Done():
+		return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
+	default:
+	}
+	var msg string
+
+	if req.Verification.Token != "" {
+		log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
+		msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+		if err != nil {
+			return err
+		}
+	}
+
+	if req.Comment.ID != "" {
+		if req.parent.User.ID == req.Comment.User.ID {
+			// don't send anything if if user replied to their own comment
+			return nil
+		}
+		log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
+		msg, err = e.buildMessageFromRequest(req)
+		if err != nil {
+			return err
+		}
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.sendMessage(emailMessage{from: e.From, to: req.Email, message: msg})
+		})
+}
+
+// buildVerificationMessage generates verification email message based on given input
+func (e *Email) buildVerificationMessage(user, email, token, site string) (string, error) {
+	subject := e.VerificationSubject
+	msg := bytes.Buffer{}
+	err := e.verifyTmpl.Execute(&msg, verifyTmplData{
+		User:         user,
+		Token:        token,
+		Email:        email,
+		Site:         site,
+		SubscribeURL: e.SubscribeURL,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build verification message")
+	}
+	return e.buildMessage(subject, msg.String(), email, "text/html", "")
+}
+
+// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
+func (e *Email) buildMessageFromRequest(req Request) (string, error) {
+	subject := "New reply to your comment"
+	if req.Comment.PostTitle != "" {
+		subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
+	}
+	token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if err != nil {
+		return "", errors.Wrapf(err, "error creating token for unsubscribe link")
+	}
+	commentUrlPrefix := req.Comment.Locator.URL + uiNav
+	msg := bytes.Buffer{}
+	err = e.msgTmpl.Execute(&msg, msgTmplData{
+		UserName:          req.Comment.User.Name,
+		UserPicture:       req.Comment.User.Picture,
+		CommentText:       req.Comment.Text,
+		CommentLink:       commentUrlPrefix + req.Comment.ID,
+		CommentDate:       req.Comment.Timestamp,
+		ParentUserName:    req.parent.User.Name,
+		ParentUserPicture: req.parent.User.Picture,
+		ParentCommentText: req.parent.Text,
+		ParentCommentLink: commentUrlPrefix + req.parent.ID,
+		ParentCommentDate: req.parent.Timestamp,
+		PostTitle:         req.Comment.PostTitle,
+		Email:             req.Email,
+		UnsubscribeLink:   unsubscribeLink,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build comment reply message")
+	}
+	return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
+}
+
+// buildMessage generates email message to send using net/smtp.Data()
+func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink string) (message string, err error) {
+	addHeader := func(msg, h, v string) string {
+		msg += fmt.Sprintf("%s: %s\n", h, v)
+		return msg
+	}
+	message = addHeader(message, "From", e.From)
+	message = addHeader(message, "To", to)
+	message = addHeader(message, "Subject", subject)
+	message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
+
+	if contentType != "" {
+		message = addHeader(message, "MIME-version", "1.0")
+		message = addHeader(message, "Content-Type", contentType+`; charset="UTF-8"`)
+	}
+
+	if unsubscribeLink != "" {
+		// https://support.google.com/mail/answer/81126 -> "Include option to unsubscribe"
+		message = addHeader(message, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
+		message = addHeader(message, "List-Unsubscribe", "<"+unsubscribeLink+">")
+	}
+
+	message = addHeader(message, "Date", time.Now().Format(time.RFC1123Z))
+
+	buff := &bytes.Buffer{}
+	qp := quotedprintable.NewWriter(buff)
+	if _, err := qp.Write([]byte(body)); err != nil {
+		return "", err
+	}
+	defer qp.Close()
+	m := buff.String()
+	message += "\n" + m
+	return message, nil
+}
+
+// sendMessage sends messages to server in a new connection, closing the connection after finishing.
+// Thread safe.
+func (e *Email) sendMessage(m emailMessage) error {
+	if e.smtp == nil {
+		return errors.New("sendMessage called without smtpClient set")
+	}
+	smtpClient, err := e.smtp.Create(e.SmtpParams)
+	if err != nil {
+		return errors.Wrap(err, "failed to make smtp Create")
+	}
+
+	defer func() {
+		if err := smtpClient.Quit(); err != nil {
+			log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
+			if err := smtpClient.Close(); err != nil {
+				log.Printf("[WARN] can't close smtp connection, %v", err)
+			}
+		}
+	}()
+
+	if err := smtpClient.Mail(m.from); err != nil {
+		return errors.Wrapf(err, "bad from address %q", m.from)
+	}
+	if err := smtpClient.Rcpt(m.to); err != nil {
+		return errors.Wrapf(err, "bad to address %q", m.to)
+	}
+
+	writer, err := smtpClient.Data()
+	if err != nil {
+		return errors.Wrap(err, "can't make email writer")
+	}
+
+	defer func() {
+		if err = writer.Close(); err != nil {
+			log.Printf("[WARN] can't close smtp body writer, %v", err)
+		}
+	}()
+
+	buf := bytes.NewBufferString(m.message)
+	if _, err = buf.WriteTo(writer); err != nil {
+		return errors.Wrapf(err, "failed to send email body to %q", m.to)
+	}
+
+	return nil
+}
+
+// String representation of Email object
+func (e *Email) String() string {
+	return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
+}
+
+// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SmtpParams
+// and returns pointer to it. Thread safe.
+func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
+	authenticate := func(c *smtp.Client) error {
+		if params.Username == "" || params.Password == "" {
+			return nil
+		}
+		auth := smtp.PlainAuth("", params.Username, params.Password, params.Host)
+		if err := c.Auth(auth); err != nil {
+			return errors.Wrapf(err, "failed to auth to smtp %s:%d", params.Host, params.Port)
+		}
+		return nil
+	}
+
+	var c *smtp.Client
+	srvAddress := fmt.Sprintf("%s:%d", params.Host, params.Port)
+	if params.TLS {
+		tlsConf := &tls.Config{
+			InsecureSkipVerify: false,
+			ServerName:         params.Host,
+		}
+		conn, err := tls.Dial("tcp", srvAddress, tlsConf)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to dial smtp tls to %s", srvAddress)
+		}
+		if c, err = smtp.NewClient(conn, params.Host); err != nil {
+			return nil, errors.Wrapf(err, "failed to make smtp client for %s", srvAddress)
+		}
+		return c, authenticate(c)
+	}
+
+	conn, err := net.DialTimeout("tcp", srvAddress, params.TimeOut)
+	if err != nil {
+		return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
+	}
+
+	c, err = smtp.NewClient(conn, srvAddress)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to dial")
+	}
+
+	return c, authenticate(c)
+}

backend/app/notify/email_test.go

@@ -0,0 +1,359 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/smtp"
+	"sync"
+	"testing"
+	"text/template"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestEmailNew(t *testing.T) {
+	var testSet = []struct {
+		name        string
+		err         bool
+		errText     string
+		emailParams EmailParams
+		smtpParams  SmtpParams
+	}{
+		{name: "empty"},
+		{name: "with template parse error",
+			err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				MsgTemplate: "{{",
+			}},
+		{name: "with verification template parse error",
+			err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From:                 "test@from",
+				VerificationTemplate: "{{",
+			},
+			smtpParams: SmtpParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+		{name: "normal creation",
+			err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From: "test@from",
+			},
+			smtpParams: SmtpParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+	}
+	for _, d := range testSet {
+		t.Run(d.name, func(t *testing.T) {
+			email, err := NewEmail(d.emailParams, d.smtpParams)
+
+			if d.err && d.errText == "" {
+				assert.Error(t, err)
+				assert.Nil(t, email)
+			} else if d.err && d.errText != "" {
+				assert.EqualError(t, err, d.errText)
+				assert.Nil(t, email)
+			} else {
+				assert.NoError(t, err)
+				assert.NotNil(t, email, "email returned")
+
+				assert.NotNil(t, email.msgTmpl, "e.template is set")
+				assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
+				assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
+				assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
+				if d.smtpParams.TimeOut == 0 {
+					assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
+				} else {
+					assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
+				}
+				assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
+				assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
+				assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
+				assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
+				assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
+			}
+		})
+	}
+}
+
+func TestEmailSendErrors(t *testing.T) {
+	var err error
+	e := Email{}
+	e.TokenGenFn = TokenGenFn
+
+	e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
+		"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
+	e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
+	assert.NoError(t, err)
+
+	e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"sending message to \"bad@example.org\" aborted due to canceled context")
+
+	e.smtp = &fakeTestSMTP{}
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
+		"error creating token for unsubscribe link: token generation error")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+}
+
+func TestEmailSend_ExitConditions(t *testing.T) {
+	email, err := NewEmail(EmailParams{}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "expecting email returned")
+	// prevent triggering e.autoFlush creation
+	emptyRequest := Request{Comment: store.Comment{ID: "999"}}
+	assert.NoError(t, email.Send(context.Background(), emptyRequest),
+		"Message without parent comment User.Email is not sent and returns nil")
+	requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
+	assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
+		"Message with parent comment User equals comment User is not sent and returns nil")
+}
+
+func TestEmailSendClientError(t *testing.T) {
+	var testSet = []struct {
+		name string
+		smtp *fakeTestSMTP
+		err  string
+	}{
+		{name: "failed to verify receiver", smtp: &fakeTestSMTP{fail: map[string]bool{"mail": true}},
+			err: "bad from address \"\": failed to verify sender"},
+		{name: "failed to verify sender", smtp: &fakeTestSMTP{fail: map[string]bool{"rcpt": true}},
+			err: "bad to address \"\": failed to verify receiver"},
+		{name: "failed to close connection", smtp: &fakeTestSMTP{fail: map[string]bool{"quit": true, "close": true}}},
+		{name: "failed to make email writer", smtp: &fakeTestSMTP{fail: map[string]bool{"data": true}},
+			err: "can't make email writer: failed to send"},
+	}
+	for _, d := range testSet {
+		t.Run(d.name, func(t *testing.T) {
+			e := Email{smtp: d.smtp}
+			if d.err != "" {
+				assert.EqualError(t, e.sendMessage(emailMessage{}), d.err,
+					"expected error for e.sendMessage")
+			} else {
+				assert.NoError(t, e.sendMessage(emailMessage{}),
+					"expected no error for e.sendMessage")
+			}
+		})
+	}
+	e := Email{}
+	e.smtp = nil
+	assert.Error(t, e.sendMessage(emailMessage{}),
+		"nil e.smtp should return error")
+	e.smtp = &fakeTestSMTP{}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage in normal flow")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"quit": true}}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage with failed smtpClient.Quit but successful smtpClient.Close")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"create": true}}
+	assert.EqualError(t, e.sendMessage(emailMessage{}), "failed to make smtp Create: failed to create client",
+		"e.send called without smtpClient set returns error")
+}
+
+func TestEmail_Send(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSmtp := fakeTestSMTP{}
+	email.smtp = &fakeSmtp
+	email.TokenGenFn = TokenGenFn
+	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
+	req := Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
+		Email:   "test@example.org",
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildMessageFromRequest(req)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: New reply to your comment for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
+Date: `)
+}
+
+func TestEmail_SendVerification(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSmtp := fakeTestSMTP{}
+	email.smtp = &fakeSmtp
+	email.TokenGenFn = TokenGenFn
+	req := Request{
+		Email: "test@example.org",
+		Verification: VerificationMetadata{
+			SiteID: "remark",
+			User:   "test_username",
+			Token:  "secret_",
+		},
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `secret_`)
+	assert.NotContains(t, res, `https://example.org/`)
+	email.SubscribeURL = "https://example.org/subscribe.html?token="
+	res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `https://example.org/subscribe.html?token=3Dsecret_`)
+}
+
+func Test_emailClient_Create(t *testing.T) {
+	creator := emailClient{}
+	client, err := creator.Create(SmtpParams{})
+	assert.Error(t, err, "absence of address to connect results in error")
+	assert.Nil(t, client, "no client returned in case of error")
+}
+
+type fakeTestSMTP struct {
+	fail map[string]bool
+
+	buff       bytes.Buffer
+	mail, rcpt string
+	auth       bool
+	close      bool
+	quitCount  int
+	lock       sync.RWMutex
+}
+
+func (f *fakeTestSMTP) Create(SmtpParams) (smtpClient, error) {
+	if f.fail["create"] {
+		return nil, errors.New("failed to create client")
+	}
+	return f, nil
+}
+
+func (f *fakeTestSMTP) Auth(smtp.Auth) error { f.auth = true; return nil }
+
+func (f *fakeTestSMTP) Mail(m string) error {
+	f.lock.Lock()
+	f.mail = m
+	f.lock.Unlock()
+	if f.fail["mail"] {
+		return errors.New("failed to verify sender")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Rcpt(r string) error {
+	f.lock.Lock()
+	f.rcpt = r
+	f.lock.Unlock()
+	if f.fail["rcpt"] {
+		return errors.New("failed to verify receiver")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Quit() error {
+	f.lock.Lock()
+	f.quitCount++
+	f.lock.Unlock()
+	if f.fail["quit"] {
+		return errors.New("failed to quit")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Close() error {
+	f.close = true
+	if f.fail["close"] {
+		return errors.New("failed to close")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Data() (io.WriteCloser, error) {
+	if f.fail["data"] {
+		return nil, errors.New("failed to send")
+	}
+	return nopCloser{&f.buff}, nil
+}
+
+func (f *fakeTestSMTP) readRcpt() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.rcpt
+}
+
+func (f *fakeTestSMTP) readMail() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.mail
+}
+
+func (f *fakeTestSMTP) readQuitCount() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.quitCount
+}
+
+func TokenGenFn(user, _, _ string) (string, error) {
+	if user == "error" {
+		return "", errors.New("token generation error")
+	}
+	return "token", nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (nopCloser) Close() error {
+	return nil
+}

backend/app/notify/notify.go

@@ -16,7 +16,7 @@ import (
 type Service struct {
 	dataService  Store
 	destinations []Destination
-	queue        chan request
+	queue        chan Request
 
 	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
 	ctx    context.Context
@@ -26,17 +26,28 @@ type Service struct {
 // Destination defines interface for a given destination service, like telegram, email and so on
 type Destination interface {
 	fmt.Stringer
-	Send(ctx context.Context, req request) error
+	Send(ctx context.Context, req Request) error
 }
 
 // Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
 }
 
-type request struct {
+// Request notification either about comment or about particular user verification
-	comment store.Comment
+type Request struct {
-	parent  store.Comment
+	Comment      store.Comment        // if set sent notifications about new comment
+	parent       store.Comment        // fetched only in case Comment is set
+	Email        string               // if set (also) send email
+	Verification VerificationMetadata // if set sent verification notification
+}
+
+// VerificationMetadata required to send notify method verification message
+type VerificationMetadata struct {
+	SiteID string
+	User   string
+	Token  string
 }
 
 const defaultQueueSize = 100
@@ -50,7 +61,7 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	ctx, cancel := context.WithCancel(context.Background())
 	res := Service{
 		dataService:  dataService,
-		queue:        make(chan request, size),
+		queue:        make(chan Request, size),
 		destinations: destinations,
 		ctx:          ctx,
 		cancel:       cancel,
@@ -62,21 +73,25 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	return &res
 }
 
-// Submit comment to internal channel if not busy, drop if can't send
+// Submit Request to internal channel if not busy, drop if can't send
-func (s *Service) Submit(comment store.Comment) {
+func (s *Service) Submit(req Request) {
 	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
 		return
 	}
-	parentComment := store.Comment{}
+	// parent comment is fetched only if comment is present in the Request
-	if s.dataService != nil {
+	if s.dataService != nil && req.Comment.ParentID != "" {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID, store.User{}); err == nil {
+		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
-			parentComment = p
+			req.parent = p
+			req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
+			if err != nil {
+				log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
+			}
 		}
 	}
 	select {
-	case s.queue <- request{comment: comment, parent: parentComment}:
+	case s.queue <- req:
 	default:
-		log.Printf("[WARN] can't send comment notification to queue, %+v", comment)
+		log.Printf("[WARN] can't send notification to queue, %+v", req.Comment)
 	}
 }
 

backend/app/notify/notify_mock.go

@@ -0,0 +1,40 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+type MockDest struct {
+	data   []Request
+	id     int
+	closed bool
+	lock   sync.Mutex
+}
+
+func (m *MockDest) Send(ctx context.Context, r Request) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.data = append(m.data, r)
+		log.Printf("sent %s -> %d", r.Comment.ID, m.id)
+	case <-ctx.Done():
+		log.Printf("ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+func (m *MockDest) Get() []Request {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]Request, len(m.data))
+	copy(res, m.data)
+	return res
+}
+func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }

backend/app/notify/notify_test.go

@@ -1,90 +1,89 @@
 package notify
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"math/rand"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestService_NoDestinations(t *testing.T) {
-	s := NewService(nil, 1)
+	s := NewService(nil, 0)
+	assert.Equal(t, defaultQueueSize, cap(s.queue))
 	assert.NotNil(t, s)
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "101"})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	assert.Equal(t, 3, len(d1.get()), "got all comments to d1")
+	require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
-	assert.Equal(t, 3, len(d2.get()), "got all comments to d2")
+	require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
 
-	assert.Equal(t, "100", d1.get()[0].comment.ID)
+	assert.Equal(t, "100", d1.Get()[0].Comment.ID)
-	assert.Equal(t, "101", d1.get()[1].comment.ID)
+	assert.Equal(t, "101", d1.Get()[1].Comment.ID)
-	assert.Equal(t, "102", d1.get()[2].comment.ID)
+	assert.Equal(t, "102", d1.Get()[2].Comment.ID)
 }
 
 func TestService_WithDrops(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
-	s.Submit(store.Comment{ID: "101"})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
-	time.Sleep(time.Millisecond * 110)
+	time.Sleep(time.Millisecond * 11)
-	s.Submit(store.Comment{ID: "102"})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
-	time.Sleep(time.Millisecond * 110)
+	time.Sleep(time.Millisecond * 11)
 	s.Close()
 
-	s.Submit(store.Comment{ID: "111"}) // safe to send after close
+	s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
 
-	assert.Equal(t, 2, len(d1.get()), "one comment dropped from d1")
+	assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
-	assert.Equal(t, 2, len(d2.get()), "one comment dropped from d2")
+	assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
 }
 
 func TestService_Many(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 5, d1, d2)
 	assert.NotNil(t, s)
 
 	for i := 0; i < 10; i++ {
-		s.Submit(store.Comment{ID: fmt.Sprintf("%d", 100+i)})
+		s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
-		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(200)))
+		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
 	}
 	s.Close()
 	time.Sleep(time.Millisecond * 10)
 
-	assert.NotEqual(t, 10, len(d1.get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
-	assert.NotEqual(t, 10, len(d2.get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
 
 	assert.True(t, d1.closed)
 	assert.True(t, d2.closed)
 }
 
 func TestService_WithParent(t *testing.T) {
-	dest := &mockDest{id: 1}
+	dest := &MockDest{id: 1}
 	dataStore := &mockStore{data: map[string]store.Comment{}}
 
 	dataStore.data["p1"] = store.Comment{ID: "p1"}
@@ -93,63 +92,37 @@ func TestService_WithParent(t *testing.T) {
 	s := NewService(dataStore, 1, dest)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "c1", ParentID: "p1"})
+	s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "c11", ParentID: "p11"})
+	s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	destRes := dest.get()
+	destRes := dest.Get()
-	assert.Equal(t, 2, len(destRes), "two comment notified")
+	require.Equal(t, 2, len(destRes), "two comment notified")
-	assert.Equal(t, "p1", destRes[0].comment.ParentID)
+	assert.Equal(t, "p1", destRes[0].Comment.ParentID)
 	assert.Equal(t, "p1", destRes[0].parent.ID)
-	assert.Equal(t, "p11", destRes[1].comment.ParentID)
+	assert.Equal(t, "p11", destRes[1].Comment.ParentID)
 	assert.Equal(t, "", destRes[1].parent.ID)
 }
 
 func TestService_Nop(t *testing.T) {
 	s := NopService
-	s.Submit(store.Comment{})
+	s.Submit(Request{Comment: store.Comment{}})
 	s.Close()
 	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
 }
 
-type mockDest struct {
-	data   []request
-	id     int
-	closed bool
-	lock   sync.Mutex
-}
-
-func (m *mockDest) Send(ctx context.Context, r request) error {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	select {
-	case <-time.After(100 * time.Millisecond):
-		m.data = append(m.data, r)
-		log.Printf("sent %s -> %d", r.comment.ID, m.id)
-	case <-ctx.Done():
-		log.Printf("ctx closed %d", m.id)
-		m.closed = true
-	}
-	return nil
-}
-
-func (m *mockDest) get() []request {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	res := make([]request, len(m.data))
-	copy(res, m.data)
-	return res
-}
-func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
-
 type mockStore struct{ data map[string]store.Comment }
 
-func (m *mockStore) Get(_ store.Locator, id string, user store.User) (store.Comment, error) {
+func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
 		return store.Comment{}, errors.New("no such id")
 	}
 	return res, nil
 }
+
+func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
+	return "", errors.New("no such user")
+}

backend/app/notify/telegram.go

@@ -40,7 +40,7 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	if res.timeout == 0 {
 		res.timeout = telegramTimeOut
 	}
-	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
+	log.Printf("[DEBUG] create new telegram notifier for chan %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -85,23 +85,27 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 }
 
 // Send to telegram channel
-func (t *Telegram) Send(ctx context.Context, req request) error {
+func (t *Telegram) Send(ctx context.Context, req Request) error {
+	if req.Comment.ID == "" {
+		// verification request received, send nothing
+		return nil
+	}
 	client := http.Client{Timeout: telegramTimeOut}
-	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
+	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
 
-	from := req.comment.User.Name
+	from := req.Comment.User.Name
-	if req.comment.ParentID != "" {
+	if req.Comment.ParentID != "" {
 		from += " → " + req.parent.User.Name
 	}
 	from = "*" + from + "*"
-	link := fmt.Sprintf("↦ [original comment](%s)", req.comment.Locator.URL+uiNav+req.comment.ID)
+	link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
-	if req.comment.PostTitle != "" {
+	if req.Comment.PostTitle != "" {
-		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
+		link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
 	}
 	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
 		t.apiPrefix, t.token, t.channelID)
 
-	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
+	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.Comment.Orig, link)
 	msg = html.UnescapeString(msg)
 	body := struct {
 		Text string `json:"text"`

backend/app/notify/telegram_test.go

@@ -30,7 +30,7 @@ func TestTelegram_New(t *testing.T) {
 	assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
 
 	_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "can't decode response:")
 
 	_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
@@ -58,24 +58,25 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NoError(t, err)
 	assert.NotNil(t, tb)
-	c := store.Comment{Text: "some text", ParentID: "1"}
+	c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
 	c.User.Name = "from"
 	cp := store.Comment{Text: "some parent text"}
 	cp.User.Name = "to"
 
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 	c.PostTitle = "test title"
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err, "should failed")
+	assert.Error(t, err, "should failed")
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
-	require.NotNil(t, err)
+	require.Error(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())
+	require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
 }
 
 func mockTelegramServer() *httptest.Server {

backend/app/rest/api/admin.go

@@ -9,18 +9,19 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
 	dataService   adminStore
-	cache         cache.LoadingCache
+	cache         LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
@@ -29,10 +30,11 @@ type admin struct {
 type adminStore interface {
 	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
 	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
 	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
 	IsBlocked(siteID string, userID string) bool
 	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
-	Blocked(siteID string) ([]store.BlockedUser, error)
+	BlockedUsers(siteID string) ([]store.BlockedUser, error)
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
 	SetVerified(siteID string, userID string, status bool) error
@@ -89,7 +91,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, ucomments[0].User)
 }
 
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
+// GET /deleteme?token=jwt - delete all user comments and details by user's request. Gets info about deleted used from provided token
 // request made GET to allow direct click from the email sent by user
 func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -109,6 +111,12 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if err := a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+
 	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
@@ -158,7 +166,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 // GET /blocked?site=siteID - list blocked users
 func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
+	users, err := a.dataService.BlockedUsers(siteID)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return

backend/app/rest/api/admin_test.go

@@ -15,8 +15,8 @@ import (
 
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,77 +30,79 @@ func TestAdmin_Delete(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
 	// check last comments
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
 
 	// check multi count
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 
 	// delete a comment
 	req, err := http.NewRequest(http.MethodDelete,
-		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	requireAdminOnly(t, req)
 	resp, err = sendReq(t, req, adminUmputunToken)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
 
 	time.Sleep(250 * time.Millisecond)
 	// check last comments updated
-	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	comments = []store.Comment{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, len(comments), "should have 1 comments")
 
 	// check count updated
-	res, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah")
+	res, code = get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah")
 	assert.Equal(t, 200, code)
 	b := map[string]interface{}{}
 	err = json.Unmarshal([]byte(res), &b)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	t.Logf("%#v", b)
 	assert.Equal(t, 1.0, b["count"], "should report 1 comments")
 
 	// check multi count updated
-	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
@@ -126,26 +128,26 @@ func TestAdmin_Title(t *testing.T) {
 	defer tss.Close()
 
 	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: tss.URL + "/post1"}}
 	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: tss.URL + "/post2"}}
 
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/title/%s?site=remark42&url=%s/post1", ts.URL, id1, tss.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
+	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=%s/post1", ts.URL, id1, tss.URL))
 	require.Equal(t, 200, code)
 	cr := store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "post1 blah 123", cr.PostTitle)
 }
 
@@ -154,11 +156,11 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	c2 := store.Comment{Text: "test test #2", Orig: "o test test #2", User: store.User{ID: "id2", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	c3 := store.Comment{Text: "test test #3", Orig: "o test test #3", User: store.User{ID: "id2", Name: "name"}, ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 
 	// write comments directly to store to keep user id
 	id1, err := srv.DataService.Create(c1)
@@ -168,20 +170,20 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	_, err = srv.DataService.Create(c3)
 	assert.NoError(t, err)
 
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
+	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42", ts.URL, "id2"), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	cmntWithInfo := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &cmntWithInfo)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
+	require.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
 
 	// id1 comment untouched
 	assert.Equal(t, id1, cmntWithInfo.Comments[0].ID)
@@ -206,9 +208,9 @@ func TestAdmin_Pin(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
@@ -216,32 +218,32 @@ func TestAdmin_Pin(t *testing.T) {
 	pin := func(val int) int {
 		client := http.Client{}
 		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
+			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=remark42&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		requireAdminOnly(t, req)
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		return resp.StatusCode
 	}
 
 	code := pin(1)
 	assert.Equal(t, 200, code)
 
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, cr.Pin)
 
 	code = pin(-1)
 	assert.Equal(t, 200, code)
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.False(t, cr.Pin)
 }
 
@@ -250,30 +252,30 @@ func TestAdmin_Block(t *testing.T) {
 	defer teardown()
 
 	makeTwoComments := func() {
-		c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-		c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 		_, err := srv.DataService.Create(c1)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		_, err = srv.DataService.Create(c2)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}
 
 	block := func(val int, ttl string) (code int, body []byte) {
-		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
+		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d", ts.URL, "user1", val)
 		if ttl != "" {
 			url = url + "&ttl=" + ttl
 		}
-		req, e := http.NewRequest(http.MethodPut, url, nil)
+		req, err := http.NewRequest(http.MethodPut, url, nil)
-		assert.Nil(t, e)
+		assert.NoError(t, err)
 		requireAdminOnly(t, req)
-		resp, e := sendReq(t, req, adminUmputunToken)
+		resp, err := sendReq(t, req, adminUmputunToken)
-		require.Nil(t, e)
+		require.NoError(t, err)
-		body, e = ioutil.ReadAll(resp.Body)
+		body, err = ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
+		assert.NoError(t, err)
-		require.Nil(t, resp.Body.Close())
+		require.NoError(t, resp.Body.Close())
 		return resp.StatusCode, body
 	}
 
@@ -284,38 +286,38 @@ func TestAdmin_Block(t *testing.T) {
 	require.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal(body, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "user1", j["user_id"])
 	assert.Equal(t, true, j["block"])
-	assert.Equal(t, "radio-t", j["site_id"])
+	assert.Equal(t, "remark42", j["site_id"])
 
-	assert.True(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.True(t, srv.adminRest.dataService.IsBlocked("remark42", "user1"))
-	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("remark42", "user2"))
 
 	// get last to confirm one comment deleted
-	bodyStr, code := get(t, ts.URL+"/api/v1/last/10?site=radio-t")
+	bodyStr, code := get(t, ts.URL+"/api/v1/last/10?site=remark42")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	assert.NoError(t, json.Unmarshal([]byte(bodyStr), &pi))
 	assert.Equal(t, 1, len(pi), "last status updated, one comment left")
 
 	// check if count call has one comment left
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah"]`)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah"]`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	body, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	pi = []store.PostInfo{}
 	err = json.Unmarshal(body, &pi)
 	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}), pi)
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, "", comments.Comments[0].Text, "permanent block clear comment")
 	assert.True(t, comments.Comments[0].Deleted, "permanent block set deleted comment's status")
 
@@ -323,7 +325,7 @@ func TestAdmin_Block(t *testing.T) {
 	code, body = block(-1, "")
 	require.Equal(t, 200, code)
 	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, false, j["block"])
 
 	// block with ttl
@@ -332,85 +334,86 @@ func TestAdmin_Block(t *testing.T) {
 	require.Equal(t, 200, code)
 
 	// get as regular user
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
+	require.Equal(t, 4, len(comments.Comments), "should have 4 comments")
 	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
 	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
-	srv.pubRest.cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
+	srv.pubRest.cache = cache.NewScache(cache.NewNopCache()) // TODO: with lru cache it won't be refreshed and invalidated for long
+	// time
 	time.Sleep(50 * time.Millisecond)
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
+	require.Equal(t, 4, len(comments.Comments), "should have 4 comments")
 	assert.Equal(t, "test test #1", comments.Comments[2].Text, "restored")
 	assert.False(t, comments.Comments[2].Deleted)
 
-	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("remark42", "user1"))
-	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("remark42", "user2"))
 }
 
 func TestAdmin_BlockedList(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2 name", ID: "user2"}}
 
 	// write comments for user1 and user2
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	// block user1
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
+		fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d", ts.URL, "user1", 1), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	res, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
+		fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d&ttl=150ms", ts.URL, "user2", 1), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	res, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, res.StatusCode)
 
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", nil)
 	require.NoError(t, err)
 	res, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(users), "two users blocked")
+	require.Equal(t, 2, len(users), "two users blocked")
 	assert.Equal(t, "user1", users[0].ID)
 	assert.Equal(t, "user1 name", users[0].Name)
 	assert.Equal(t, "user2", users[1].ID)
 	assert.Equal(t, "user2 name", users[1].Name)
 	t.Logf("%+v", users)
-	time.Sleep(50 * time.Millisecond)
+	time.Sleep(150 * time.Millisecond)
 
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", nil)
 	require.NoError(t, err)
 	res, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, len(users), "one user left blocked")
 }
 
@@ -418,39 +421,39 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	info, err := srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.False(t, info.ReadOnly)
 
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := sendReq(t, req, "") // non-admin user
 	require.NoError(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// try to write comment
 	c := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	b, err := json.Marshal(c)
-	assert.Nil(t, err, "can't marshal comment %+v", c)
+	assert.NoError(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
 	require.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
@@ -459,20 +462,20 @@ func TestAdmin_ReadOnly(t *testing.T) {
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.False(t, info.ReadOnly)
 
 	// try to write comment
 	c = store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	b, err = json.Marshal(c)
-	assert.Nil(t, err, "can't marshal comment %+v", c)
+	assert.NoError(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
 	require.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
@@ -486,20 +489,20 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&format=tree")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&format=tree")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 	assert.True(t, comments.Info.ReadOnly)
 	t.Logf("%+v", comments)
@@ -509,36 +512,36 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"},
 		Timestamp: time.Date(2001, 1, 1, 1, 1, 1, 0, time.Local)}
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 10)
+	info, err := srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 10)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly, "ro by age")
 
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
@@ -547,53 +550,53 @@ func TestAdmin_Verify(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	verified := srv.DataService.IsVerified("radio-t", "user1")
+	verified := srv.DataService.IsVerified("remark42", "user1")
 	assert.False(t, verified)
 
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=remark42&verified=1", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
+	verified = srv.DataService.IsVerified("remark42", "user1")
 	assert.True(t, verified)
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, "test test #1", comments.Comments[0].Text)
 	assert.True(t, comments.Comments[0].User.Verified)
 
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=remark42&verified=0", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = sendReq(t, req, adminUmputunToken)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
+	verified = srv.DataService.IsVerified("remark42", "user1")
 	assert.False(t, verified)
 
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, "test test #1", comments.Comments[0].Text)
 	assert.False(t, comments.Comments[0].User.Verified)
 }
@@ -603,14 +606,14 @@ func TestAdmin_ExportStream(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	body, code := getWithAdminAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
+	body, code := getWithAdminAuth(t, ts.URL+"/api/v1/admin/export?site=remark42&mode=stream")
 	assert.Equal(t, 200, code)
 	assert.Equal(t, 3, strings.Count(body, "\n"))
 	assert.Equal(t, 2, strings.Count(body, "\"text\""))
@@ -622,14 +625,14 @@ func TestAdmin_ExportFile(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42&mode=file", nil)
 	require.NoError(t, err)
 	requireAdminOnly(t, req)
 	resp, err := sendReq(t, req, adminUmputunToken)
@@ -651,24 +654,32 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	comments, err := srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
+	comments, err := srv.DataService.User("remark42", "user1", 0, 0, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, len(comments), "a comment for user1")
 
+	email, err := srv.DataService.SetUserEmail("remark42", "user1", "test@example.org")
+	assert.NoError(t, err)
+	assert.Equal(t, "test@example.org", email, "new email for user1")
+
+	email, err = srv.DataService.GetUserEmail("remark42", "user1")
+	assert.NoError(t, err)
+	assert.Equal(t, "test@example.org", email, "new email for user1 is readable")
+
 	claims := token.Claims{
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
-			Audience:  "radio-t",
+			Audience:  "remark42",
 			Id:        "1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
@@ -683,53 +694,58 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 		},
 	}
 
-	require.NoError(t, os.MkdirAll("/tmp/ava-remark42/42", 0700))
+	require.NoError(t, os.MkdirAll(os.TempDir()+"/ava-remark42/42", 0700))
-	require.NoError(t, ioutil.WriteFile("/tmp/ava-remark42/42/pic.image", []byte("some image data"), 0600))
+	require.NoError(t, ioutil.WriteFile(os.TempDir()+"/ava-remark42/42/pic.image", []byte("some image data"), 0600))
 
 	tkn, err := srv.Authenticator.TokenService().Token(claims)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	_, err = srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
+	_, err = srv.DataService.User("remark42", "user1", 0, 0, store.User{})
 	assert.EqualError(t, err, "no comments for user user1 in store")
+
+	email, err = srv.DataService.GetUserEmail("remark42", "user1")
+	assert.NoError(t, err)
+	assert.Empty(t, email, "user1 email was deleted")
+
 }
 
 func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	// try with bad token
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, "bad token"), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
 
 	// try with bad auth
 	claims := token.Claims{
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
-			Audience:  "radio-t",
+			Audience:  "remark42",
 			Id:        "1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
@@ -744,39 +760,39 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	}
 
 	tkn, err := srv.Authenticator.TokenService().Token(claims)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "bad-password")
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
 
 	// try bad user
 	badClaims := claims
 	badClaims.User.ID = "no-such-id"
 	tkn, err = srv.Authenticator.TokenService().Token(badClaims)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode, resp.Status)
 
 	// try without deleteme flag
 	badClaims2 := claims
 	badClaims2.User.SetBoolAttr("delete_me", false)
 	tkn, err = srv.Authenticator.TokenService().Token(badClaims2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, strings.Contains(string(b), "can't use provided token"))
 }
 
@@ -784,27 +800,28 @@ func TestAdmin_GetUserInfo(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
 	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	body, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	body, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=remark42&url=https://radio-t.com/blah",
+		ts.URL))
 	assert.Equal(t, 200, code)
 	u := store.User{}
 	err = json.Unmarshal([]byte(body), &u)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "823688dafca7393d24c871a2da98a84d8732e927",
 		Admin: false, Blocked: false, Verified: false}, u)
 
-	_, code = get(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	_, code = get(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=remark42&url=https://radio-t.com/blah", ts.URL))
 	assert.Equal(t, 401, code, "no auth")
 
-	_, code = getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/userX?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	_, code = getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/userX?site=remark42&url=https://radio-t.com/blah", ts.URL))
 	assert.Equal(t, 400, code, "no info about user")
 }

backend/app/rest/api/migrator.go

@@ -12,9 +12,9 @@ import (
 	"time"
 
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/migrator"
@@ -23,11 +23,12 @@ import (
 
 // Migrator rest with import and export controllers
 type Migrator struct {
-	Cache             cache.LoadingCache
+	Cache             LoadingCache
 	NativeImporter    migrator.Importer
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
 	NativeExporter    migrator.Exporter
+	UrlMapperMaker    migrator.MapperMaker
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -98,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"status": "import request accepted"})
 }
 
-func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /wait?site=site-id
+// waits for migration operation (import or remap)
+func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	timeOut := time.Minute * 15
 	if v := r.URL.Query().Get("timeout"); v != "" {
@@ -152,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// POST /remap?site=site-id
+// remap urls in comments based on given rules (oldUrl newUrl)
+func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	// create new url-mapper from given rules in body
+	mapper, err := m.UrlMapperMaker(r.Body)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
+		return
+	}
+	defer r.Body.Close()
+
+	// start remap procedure with mapper
+	go func() {
+		m.setBusy(siteID, true)
+		defer m.setBusy(siteID, false)
+
+		// do export
+		fh, e := ioutil.TempFile("", "remark42_convert")
+		if e != nil {
+			log.Printf("[WARN] failed to make temp file %+v", e)
+			return
+		}
+		defer func() {
+			if e := os.Remove(fh.Name()); e != nil {
+				log.Printf("[WARN] failed to remove temp file %+v", e)
+			}
+		}()
+		log.Printf("[DEBUG] start export for site=%s", siteID)
+		if _, e := m.NativeExporter.Export(fh, siteID); e != nil {
+			log.Printf("[WARN] export failed with %+v", e)
+			return
+		}
+
+		if _, e = fh.Seek(0, 0); e != nil {
+			log.Printf("[WARN] failed to seek file %+v", e)
+			return
+		}
+
+		log.Printf("[DEBUG] start import for site=%s", siteID)
+		mappedReader := migrator.WithMapper(fh, mapper)
+		size, e := m.NativeImporter.Import(mappedReader, siteID)
+		if e != nil {
+			log.Printf("[WARN] import failed with %+v", e)
+			return
+		}
+
+		m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
+		log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size)
+	}()
+
+	render.Status(r, http.StatusAccepted)
+	render.JSON(w, r, R.JSON{"status": "convert request accepted"})
+}
+
 // runImport reads from tmpfile and import for given siteID and provider
 func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	m.setBusy(siteID, true)

backend/app/rest/api/migrator_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -15,37 +16,52 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
@@ -57,15 +73,15 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=remark42&provider=native", contentType, bodyBuf)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
@@ -75,33 +91,39 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=wordpress", r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
@@ -109,53 +131,59 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 150; i++ {
+	for i := 0; i < 50; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 1 * time.Second}
+	client = &http.Client{Timeout: 5 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 150; i++ {
+	for i := 0; i < 50; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
-	client := &http.Client{Timeout: 1 * time.Second}
+	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 10 * time.Second}
+	client = &http.Client{Timeout: 5 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t&timeout=100ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=10ms", nil)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
@@ -163,32 +191,38 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
@@ -201,11 +235,11 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
 
@@ -215,26 +249,123 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+func TestMigrator_Remap(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create 2 comments in https://remark42.com/demo/
+	c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
+	_, err := srv.DataService.Create(c1)
+	require.NoError(t, err)
+	c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
+	_, err = srv.DataService.Create(c2)
+	require.NoError(t, err)
+
+	// create 1 comment in https://remark42.com/demo-another/
+	c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
+	_, err = srv.DataService.Create(c3)
+	require.NoError(t, err)
+
+	// set url https://remark42.com/demo-another/ to be readonly
+	err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
+		URL:      "https://remark42.com/demo-another/",
+		ReadOnly: true,
+	}})
+	require.NoError(t, err)
+
+	// check that comments created as expected
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// we want remap urls to another domain - www.remark42.com
+	rules := "https://remark42.com/* https://www.remark42.com/*"
+	resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	waitForMigrationCompletion(t, ts)
+
+	// after remap finished we should find comments from new urls
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// should find nothing from previous url
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+}
+
+func TestMigrator_RemapReject(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// without admin credentials
+	client := &http.Client{Timeout: 1 * time.Second}
+	rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
+	require.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil)
 	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
 	resp, err := client.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
+	assert.Equal(t, "{\"site_id\":\"remark42\",\"status\":\"completed\"}\n", string(b))
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -17,9 +18,9 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/go-pkgz/rest/logger"
 	"github.com/pkg/errors"
 	"github.com/rakyll/statik/fs"
@@ -38,7 +39,7 @@ type Rest struct {
 
 	DataService      *service.DataStore
 	Authenticator    *auth.Service
-	Cache            cache.LoadingCache
+	Cache            LoadingCache
 	ImageProxy       *proxy.Image
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
@@ -46,6 +47,7 @@ type Rest struct {
 	ImageService     *image.Service
 	Streamer         *Streamer
 
+	AnonVote        bool
 	WebRoot         string
 	RemarkURL       string
 	ReadOnlyAge     int
@@ -54,8 +56,10 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter float64
+	UpdateLimiter      float64
-	EmojiEnabled  bool
+	EmailNotifications bool
+	EmojiEnabled       bool
+	SimpleView         bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
@@ -68,6 +72,12 @@ type Rest struct {
 	rssRest   rss
 }
 
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req lcw.FlusherRequest)                                        // evict matched records
+}
+
 const hardBodyLimit = 1024 * 64 // limit size of body
 
 const lastCommentsScope = "last"
@@ -258,7 +268,7 @@ func (s *Rest) routes() chi.Router {
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(middleware.Timeout(30 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth, middleware.NoCache, logInfoWithBody)
+			rauth.Use(authMiddleware.Auth, matchSiteID, middleware.NoCache, logInfoWithBody)
 			rauth.Get("/user", s.privRest.userInfoCtrl)
 			rauth.Get("/userdata", s.privRest.userAllDataCtrl)
 		})
@@ -267,7 +277,7 @@ func (s *Rest) routes() chi.Router {
 		rapi.Route("/admin", func(radmin chi.Router) {
 			radmin.Use(middleware.Timeout(30 * time.Second))
 			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly)
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly, matchSiteID)
 			radmin.Use(middleware.NoCache, logInfoWithBody)
 
 			radmin.Delete("/comment/{id}", s.adminRest.deleteCommentCtrl)
@@ -285,28 +295,33 @@ func (s *Rest) routes() chi.Router {
 			radmin.Get("/export", s.adminRest.migrator.exportCtrl)
 			radmin.Post("/import", s.adminRest.migrator.importCtrl)
 			radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
-			radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl)
+			radmin.Post("/remap", s.adminRest.migrator.remapCtrl)
+			radmin.Get("/wait", s.adminRest.migrator.waitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(middleware.Timeout(10 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(authMiddleware.Auth, matchSiteID)
 			rauth.Use(middleware.NoCache)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
 			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
 			rauth.Post("/comment", s.privRest.createCommentCtrl)
-			rauth.With(rejectAnonUser).Put("/vote/{id}", s.privRest.voteCtrl)
+			rauth.Put("/vote/{id}", s.privRest.voteCtrl)
 			rauth.With(rejectAnonUser).Post("/deleteme", s.privRest.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Get("/email", s.privRest.getEmailCtrl)
+			rauth.With(rejectAnonUser).Post("/email/subscribe", s.privRest.sendEmailConfirmationCtrl)
+			rauth.With(rejectAnonUser).Post("/email/confirm", s.privRest.setConfirmedEmailCtrl)
+			rauth.With(rejectAnonUser).Delete("/email", s.privRest.deleteEmailCtrl)
 		})
 
 		// protected routes, anonymous rejected
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(middleware.Timeout(10 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
-			rauth.Use(authMiddleware.Auth, rejectAnonUser)
+			rauth.Use(authMiddleware.Auth, rejectAnonUser, matchSiteID)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 			rauth.Post("/picture", s.privRest.savePictureCtrl)
 		})
@@ -319,10 +334,12 @@ func (s *Rest) routes() chi.Router {
 		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
 		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
 		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
+		rroot.Get("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
+		rroot.Post("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
 	})
 
 	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
+	addFileServer(router, "/web", http.Dir(s.WebRoot), s.Version)
 	return router
 }
 
@@ -347,6 +364,7 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 		authenticator:    s.Authenticator,
 		notifyService:    s.NotifyService,
 		remarkURL:        s.RemarkURL,
+		anonVote:         s.AnonVote,
 	}
 
 	admGrp := admin{
@@ -382,30 +400,36 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 	emails, _ := s.DataService.AdminStore.Email(siteID)
 
 	cnf := struct {
-		Version        string   `json:"version"`
+		Version            string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
+		EditDuration       int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
+		MaxCommentSize     int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
+		Admins             []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
+		AdminEmail         string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
+		Auth               []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
+		AnonVote           bool     `json:"anon_vote"`
-		CriticalScore  int      `json:"critical_score"`
+		LowScore           int      `json:"low_score"`
-		PositiveScore  bool     `json:"positive_score"`
+		CriticalScore      int      `json:"critical_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
+		PositiveScore      bool     `json:"positive_score"`
-		MaxImageSize   int      `json:"max_image_size"`
+		ReadOnlyAge        int      `json:"readonly_age"`
-		EmojiEnabled   bool     `json:"emoji_enabled"`
+		MaxImageSize       int      `json:"max_image_size"`
+		EmailNotifications bool     `json:"email_notifications"`
+		EmojiEnabled       bool     `json:"emoji_enabled"`
+		SimpleView         bool     `json:"simple_view"`
 	}{
-		Version:        s.Version,
+		Version:            s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
+		EditDuration:       int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
+		MaxCommentSize:     s.DataService.MaxCommentSize,
-		Admins:         admins,
+		Admins:             admins,
-		AdminEmail:     emails,
+		AdminEmail:         emails,
-		LowScore:       s.ScoreThresholds.Low,
+		LowScore:           s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
+		CriticalScore:      s.ScoreThresholds.Critical,
-		PositiveScore:  s.DataService.PositiveScore,
+		PositiveScore:      s.DataService.PositiveScore,
-		ReadOnlyAge:    s.ReadOnlyAge,
+		ReadOnlyAge:        s.ReadOnlyAge,
-		MaxImageSize:   s.ImageService.Store.SizeLimit(),
+		MaxImageSize:       s.ImageService.Store.SizeLimit(),
-		EmojiEnabled:   s.EmojiEnabled,
+		EmailNotifications: s.EmailNotifications,
+		EmojiEnabled:       s.EmojiEnabled,
+		AnonVote:           s.AnonVote,
+		SimpleView:         s.SimpleView,
 	}
 
 	cnf.Auth = []string{}
@@ -421,7 +445,7 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // serves static files from /web or embedded by statik
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
+func addFileServer(r chi.Router, path string, root http.FileSystem, version string) {
 
 	var webFS http.Handler
 
@@ -443,15 +467,17 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	}
 	path += "*"
 
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)), middleware.Timeout(10*time.Second)).
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)),
-		Get(path, func(w http.ResponseWriter, r *http.Request) {
+		middleware.Timeout(10*time.Second),
-			// don't show dirs, just serve files
+		cacheControl(time.Hour, version),
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
+	).Get(path, func(w http.ResponseWriter, r *http.Request) {
-				http.NotFound(w, r)
+		// don't show dirs, just serve files
-				return
+		if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-			}
+			http.NotFound(w, r)
-			webFS.ServeHTTP(w, r)
+			return
-		})
+		}
+		webFS.ServeHTTP(w, r)
+	})
 }
 
 func encodeJSONWithHTML(v interface{}) ([]byte, error) {
@@ -518,6 +544,57 @@ func rejectAnonUser(next http.Handler) http.Handler {
 	return http.HandlerFunc(fn)
 }
 
+// matchSiteID is a middleware rejecting users with mismatch between site param and and User.SiteID
+func matchSiteID(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		// skip for basic auth user
+		if user.Name == "admin" && user.ID == "admin" {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		siteID := r.URL.Query().Get("site")
+		if siteID != "" && user.SiteID != siteID {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
+// cacheControl is a middleware setting cache expiration. Using url+version as etag
+func cacheControl(expiration time.Duration, version string) func(http.Handler) http.Handler {
+
+	etag := func(r *http.Request, version string) string {
+		s := version + ":" + r.URL.String()
+		return store.EncodeID(s)
+	}
+
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+			e := `"` + etag(r, version) + `"`
+			w.Header().Set("Etag", e)
+			w.Header().Set("Cache-Control", "max-age="+strconv.Itoa(int(expiration.Seconds())))
+
+			if match := r.Header.Get("If-None-Match"); match != "" {
+				if strings.Contains(match, e) {
+					w.WriteHeader(http.StatusNotModified)
+					return
+				}
+			}
+			h.ServeHTTP(w, r)
+		}
+		return http.HandlerFunc(fn)
+	}
+}
+
 func parseError(err error, defaultCode int) (code int) {
 	code = defaultCode
 

backend/app/rest/api/rest_private.go

@@ -1,10 +1,13 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -14,35 +17,40 @@ import (
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/hashicorp/go-multierror"
 
 	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
 type private struct {
 	dataService      privStore
-	cache            cache.LoadingCache
+	cache            LoadingCache
 	readOnlyAge      int
 	commentFormatter *store.CommentFormatter
 	imageService     *image.Service
 	notifyService    *notify.Service
 	authenticator    *auth.Service
 	remarkURL        string
+	anonVote         bool
 }
 
 type privStore interface {
 	Create(comment store.Comment) (commentID string, err error)
 	EditComment(locator store.Locator, commentID string, req service.EditRequest) (comment store.Comment, err error)
-	Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error)
+	Vote(req service.VoteReq) (comment store.Comment, err error)
 	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
 	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
+	SetUserEmail(siteID string, userID string, value string) (string, error)
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
 	ValidateComment(c *store.Comment) error
 	IsVerified(siteID string, userID string) bool
 	IsReadOnly(locator store.Locator) bool
@@ -50,6 +58,21 @@ type privStore interface {
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 }
 
+const unsubscribeHtml = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Successfully unsubscribed</p>
+</div>
+</body>
+</html>
+`
+
 // POST /comment - adds comment, resets all immutable fields
 func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -60,6 +83,12 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	user := rest.MustGetUserInfo(r)
+	if user.ID != "admin" && user.SiteID != comment.Locator.SiteID {
+		rest.SendErrorJSON(w, r, http.StatusForbidden,
+			fmt.Errorf("site mismatch, %q not allowed to post to %s", user.SiteID, comment.Locator.SiteID), "invalid site",
+			rest.ErrCommentValidation)
+		return
+	}
 
 	comment.PrepareUntrusted() // clean all fields user not supposed to set
 	comment.User = user
@@ -103,7 +132,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
 	if s.notifyService != nil {
-		s.notifyService.Submit(finalComment)
+		s.notifyService.Submit(notify.Request{Comment: finalComment})
 	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -173,6 +202,14 @@ func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
 		user.Verified = s.dataService.IsVerified(siteID, user.ID)
+
+		email, err := s.dataService.GetUserEmail(siteID, user.ID)
+		if err != nil {
+			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+		}
+		if len(email) > 0 {
+			user.EmailSubscription = true
+		}
 	}
 
 	render.JSON(w, r, user)
@@ -181,6 +218,10 @@ func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
 func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
+	if !s.anonVote && strings.HasPrefix(user.ID, "anonymous_") {
+		http.Error(w, "Access denied", http.StatusForbidden)
+		return
+	}
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
 	log.Printf("[DEBUG] vote for comment %s", id)
@@ -198,7 +239,14 @@ func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comment, err := s.dataService.Vote(locator, id, user.ID, vote)
+	req := service.VoteReq{
+		Locator:   locator,
+		CommentID: id,
+		UserID:    user.ID,
+		UserIP:    strings.Split(r.RemoteAddr, ":")[0],
+		Val:       vote,
+	}
+	comment, err := s.dataService.Vote(req)
 	if err != nil {
 		code := parseError(err, rest.ErrVoteRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
@@ -208,6 +256,220 @@ func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
+// getEmailCtrl gets email address for authenticated user.
+// GET /email?site=siteID
+func (s *private) getEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	address, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// sendEmailConfirmationCtrl gets address and siteID from query, makes confirmation token and sends it to user.
+// GET /email/subscribe?site=siteID&address=someone@example.com
+func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	address := r.URL.Query().Get("address")
+	siteID := r.URL.Query().Get("site")
+	if address == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
+		return
+	}
+	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+	if address == existingAddress {
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
+		return
+	}
+	claims := token.Claims{
+		Handshake: &token.Handshake{ID: user.ID + "::" + address},
+		StandardClaims: jwt.StandardClaims{
+			Audience:  r.URL.Query().Get("site"),
+			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
+			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+			Issuer:    "remark42",
+		},
+	}
+
+	tkn, err := s.authenticator.TokenService().Token(claims)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to make verification token", rest.ErrInternal)
+		return
+	}
+
+	s.notifyService.Submit(
+		notify.Request{
+			Email: address,
+			Verification: notify.VerificationMetadata{
+				SiteID: siteID,
+				User:   user.Name,
+				Token:  tkn,
+			},
+		},
+	)
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// setConfirmedEmailCtrl uses provided token parameter (generated by sendEmailConfirmationCtrl) to set email and add it to user token
+// PUT /email/confirm?site=siteID&tkn=jwt
+func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 || elems[0] != user.ID {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	address := elems[1]
+
+	log.Printf("[DEBUG] set email for user %s", user.ID)
+
+	val, err := s.dataService.SetUserEmail(siteID, user.ID, address)
+	if err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set email for user", code)
+		return
+	}
+
+	// update User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	claims.User.Email = address
+	if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+		return
+	}
+	render.JSON(w, r, R.JSON{"updated": true, "address": val})
+}
+
+// POST/GET /email/unsubscribe.html?site=siteID&tkn=jwt - unsubscribe the user in token from email notifications
+func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	userID := elems[0]
+	address := elems[1]
+
+	existingAddress, err := s.dataService.GetUserEmail(siteID, userID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", userID, err)
+	}
+	if existingAddress == "" {
+		rest.SendErrorHTML(w, r, http.StatusConflict, errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
+		return
+	}
+	if address != existingAddress {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
+		return
+	}
+
+	log.Printf("[DEBUG] unsubscribe user %s", userID)
+
+	if err := s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token, if user has the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		log.Printf("[DEBUG] unsubscribed user doesn't have valid JWT token to update %s, %v", userID, err)
+	}
+	if claims.User != nil && claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err := tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHtml))
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, nil)
+	render.HTML(w, r, msg.String())
+}
+
+// DELETE /email?site=siteID - removes user's email
+func (s *private) deleteEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] remove email for user %s", user.ID)
+
+	if err := s.dataService.DeleteUserDetail(siteID, user.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	if claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+	render.JSON(w, r, R.JSON{"deleted": true})
+}
+
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
 func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")

backend/app/rest/api/rest_private_test.go

@@ -15,11 +15,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/dgrijalva/jwt-go"
+	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth/token"
 	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/image"
 )
@@ -34,18 +38,17 @@ func TestRest_Create(t *testing.T) {
 	defer teardown()
 
 	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
 
-	t.Log(string(b))
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	loc := c["locator"].(map[string]interface{})
-	assert.Equal(t, "radio-t", loc["site"])
+	assert.Equal(t, "remark42", loc["site"])
 	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
 	assert.True(t, len(c["id"].(string)) > 8)
 }
@@ -56,30 +59,30 @@ func TestRest_CreateOldPost(t *testing.T) {
 
 	// make old, but not too old comment
 	old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(old)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time", store.User{})
+	comments, err := srv.DataService.Find(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, "time", store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, len(comments))
 
 	// try to add new comment to the same old post
 	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
+		`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 
-	assert.Nil(t, srv.DataService.DeleteAll("radio-t"))
+	assert.NoError(t, srv.DataService.DeleteAll("remark42"))
 	// make too old comment
 	old = store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(old)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	resp, err = post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
+		`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 }
 
@@ -87,28 +90,28 @@ func TestRest_CreateTooBig(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
+	longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`, "Щ")
 
 	resp, err := post(t, ts.URL+"/api/v1/comment", longComment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "comment text exceeded max allowed size 4000 (4001)", c["error"])
 	assert.Equal(t, "invalid comment", c["details"])
 
-	veryLongComment := fmt.Sprintf(`{"text": "%70000s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
+	veryLongComment := fmt.Sprintf(`{"text": "%70000s", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`, "Щ")
 	resp, err = post(t, ts.URL+"/api/v1/comment", veryLongComment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	c = R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "http: request body too large", c["error"])
 	assert.Equal(t, "can't bind comment", c["details"])
 }
@@ -117,16 +120,17 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1", 
+"site": "remark42"}}`)
 
 	resp, err := post(t, ts.URL+"/api/v1/comment", badComment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "comment contains restricted words", c["error"])
 	assert.Equal(t, "invalid comment", c["details"])
 }
@@ -135,12 +139,21 @@ func TestRest_CreateRejected(t *testing.T) {
 
 	ts, _, teardown := startupT(t)
 	defer teardown()
-	body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`
+	body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
 
 	// try to create without auth
 	resp, err := http.Post(ts.URL+"/api/v1/comment", "", strings.NewReader(body))
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
+
+	// try with wrong aud
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devTokenBadAud)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusForbidden, resp.StatusCode, "reject wrong aud")
 }
 
 func TestRest_CreateAndGet(t *testing.T) {
@@ -149,36 +162,35 @@ func TestRest_CreateAndGet(t *testing.T) {
 
 	// create comment
 	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "**test** *123*\n\n http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+		`{"text": "**test** *123*\n\n http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	id := c["id"].(string)
 
 	// get created comment by id as admin
-	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	comment := store.Comment{}
 	err = json.Unmarshal([]byte(res), &comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<p><strong>test</strong> <em>123</em></p>\n\n<p><a href=\"http://radio-t.com\" rel=\"nofollow\">http://radio-t.com</a></p>\n", comment.Text)
 	assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig)
 	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false,
 		IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
 		comment.User)
-	t.Logf("%+v", comment)
 
 	// get created comment by id as non-admin
-	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	comment = store.Comment{}
 	err = json.Unmarshal([]byte(res), &comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false, IP: ""}, comment.User, "no ip")
 }
 
@@ -187,24 +199,24 @@ func TestRest_Update(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
 	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=remark42&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 200, b.StatusCode, string(body))
 
 	// comments returned by update
 	c2 := store.Comment{}
 	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, id, c2.ID)
 	assert.Equal(t, "<p>updated text</p>\n", c2.Text)
 	assert.Equal(t, "updated text", c2.Orig)
@@ -212,11 +224,11 @@ func TestRest_Update(t *testing.T) {
 	assert.True(t, time.Since(c2.Edit.Timestamp) < 1*time.Second)
 
 	// read updated comment
-	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	c3 := store.Comment{}
 	err = json.Unmarshal([]byte(res), &c3)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, c2, c3, "same as response from update")
 }
 
@@ -225,24 +237,24 @@ func TestRest_UpdateDelete(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
 	// check multi count updated
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	j := []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 
 	// delete a comment
 	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=remark42&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
@@ -260,21 +272,21 @@ func TestRest_UpdateDelete(t *testing.T) {
 	assert.True(t, c2.Deleted)
 
 	// read updated comment
-	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	c3 := store.Comment{}
 	err = json.Unmarshal([]byte(res), &c3)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "", c3.Text)
 	assert.Equal(t, "", c3.Orig)
 	assert.True(t, c3.Deleted)
 
 	// check multi count updated
-	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	bb, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	j = []store.PostInfo{}
 	err = json.Unmarshal(bb, &j)
 	require.NoError(t, err)
@@ -287,52 +299,70 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
 	id1, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
+		"?site=remark42&url=https://radio-t.com/blah1", strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
 	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
 	client = http.Client{}
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
+		"?site=remark42&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
 }
 
+func TestRest_UpdateWrongAud(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	id := addComment(t, c1, ts)
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=remark42&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
+	assert.NoError(t, err)
+	req.Header.Add("X-JWT", devTokenBadAud)
+	b, err := client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, b.StatusCode, "reject update with wrong aut in jwt")
+}
+
 func TestRest_UpdateWithRestrictedWords(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	c1 := store.Comment{Text: "What the quack is that?", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
 	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=remark42&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"text":"What the duck is that?", "summary":"my edit"}`))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	c := R.JSON{}
 	err = json.Unmarshal(body, &c)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 400, b.StatusCode, string(body))
 	assert.Equal(t, "comment contains restricted words", c["error"])
 	assert.Equal(t, "invalid comment", c["details"])
@@ -343,9 +373,9 @@ func TestRest_Vote(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
 
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
@@ -353,102 +383,355 @@ func TestRest_Vote(t *testing.T) {
 	vote := func(val int) int {
 		client := http.Client{}
 		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
+			fmt.Sprintf("%s/api/v1/vote/%s?site=remark42&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		return resp.StatusCode
 	}
 
 	assert.Equal(t, 200, vote(1), "first vote allowed")
 	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, cr.Score)
 	assert.Equal(t, 1, cr.Vote)
 	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, cr.Score)
 	assert.Equal(t, 0, cr.Vote)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed one more time")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, -1, cr.Vote)
 
 	assert.Equal(t, 400, vote(-1), "dbl vote not allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, -1, cr.Vote)
 
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
 	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	req, err := http.NewRequest("GET",
-		fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+		fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.NoError(t, err)
 	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	cr = store.Comment{}
 	err = json.NewDecoder(resp.Body).Decode(&cr)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, -1, cr.Score)
 	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
 	assert.Equal(t, map[string]bool(nil), cr.Votes)
+}
 
+func TestRest_AnonVote(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}}
+
+	id1 := addComment(t, c1, ts)
+	addComment(t, c2, ts)
+
+	vote := func(val int) int {
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPut,
+			fmt.Sprintf("%s/api/v1/vote/%s?site=remark42&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
+		assert.NoError(t, err)
+		req.Header.Add("X-JWT", anonToken)
+		resp, err := client.Do(req)
+		assert.NoError(t, err)
+		return resp.StatusCode
+	}
+
+	getWithAnonAuth := func(url string) (body string, code int) {
+		client := &http.Client{Timeout: 5 * time.Second}
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(t, err)
+		req.Header.Add("X-JWT", anonToken)
+		r, err := client.Do(req)
+		require.NoError(t, err)
+		defer r.Body.Close()
+		b, err := ioutil.ReadAll(r.Body)
+		assert.NoError(t, err)
+		return string(b), r.StatusCode
+	}
+
+	assert.Equal(t, 403, vote(1), "vote is disallowed with anonVote false")
+	srv.privRest.anonVote = true
+	assert.Equal(t, 200, vote(1), "first vote allowed")
+	assert.Equal(t, 400, vote(1), "second vote rejected")
+	body, code := getWithAnonAuth(fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+
+	assert.Equal(t, 200, code)
+	cr := store.Comment{}
+	err := json.Unmarshal([]byte(body), &cr)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, cr.Score)
+	assert.Equal(t, 1, cr.Vote)
 	assert.Equal(t, map[string]bool(nil), cr.Votes)
 }
 
+func TestRest_Email(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// issue good token
+	claims := token.Claims{
+		Handshake: &token.Handshake{ID: "dev::good@example.com"},
+		StandardClaims: jwt.StandardClaims{
+			Audience:  "remark42",
+			ExpiresAt: time.Now().Add(10 * time.Minute).Unix(),
+			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+			Issuer:    "remark42",
+		},
+	}
+	tkn, err := srv.Authenticator.TokenService().Token(claims)
+	require.NoError(t, err)
+	goodToken := tkn
+
+	var testData = []struct {
+		description  string
+		url          string
+		method       string
+		responseCode int
+		noAuth       bool
+		cookieEmail  string
+	}{
+		{description: "issue delete request without auth", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusUnauthorized, noAuth: true},
+		{description: "issue delete request without site_id", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusBadRequest},
+		{description: "delete non-existent user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
+		{description: "set user email, token not set", url: "/api/v1/email/confirm?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send confirmation without address", url: "/api/v1/email/subscribe?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
+		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "send confirmation with same address", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusConflict},
+		{description: "get user email", url: "/api/v1/email?site=remark42", method: http.MethodGet, responseCode: http.StatusOK},
+		{description: "delete user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
+		{description: "send another confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
+		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "unsubscribe user, no token", url: "/email/unsubscribe.html?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "unsubscribe user, wrong token", url: "/email/unsubscribe.html?site=remark42&tkn=jwt", method: http.MethodGet, responseCode: http.StatusForbidden},
+		{description: "unsubscribe user, good token", url: fmt.Sprintf("/email/unsubscribe.html?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK},
+		{description: "unsubscribe user second time, good token", url: fmt.Sprintf("/email/unsubscribe.html?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusConflict},
+	}
+	client := http.Client{}
+	for _, x := range testData {
+		t.Run(x.description, func(t *testing.T) {
+			req, err := http.NewRequest(x.method, ts.URL+x.url, nil)
+			require.NoError(t, err)
+			if !x.noAuth {
+				req.Header.Add("X-JWT", devToken)
+			}
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			// read User.Email from the token in the cookie
+			for _, c := range resp.Cookies() {
+				if c.Name == "JWT" {
+					claims, err := srv.Authenticator.TokenService().Parse(c.Value)
+					require.NoError(t, err)
+					assert.Equal(t, x.cookieEmail, claims.User.Email, "cookie email check failed")
+				}
+			}
+			assert.Equal(t, x.responseCode, resp.StatusCode, string(body))
+		})
+	}
+}
+
+func TestRest_EmailNotification(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	mockDestination := &notify.MockDest{}
+	srv.privRest.notifyService = notify.NewService(srv.DataService, 1, mockDestination)
+
+	client := http.Client{}
+
+	// create new comment from dev user
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
+		`{"text": "test 123",
+"user": {"name": "dev::good@example.com"},
+"locator":{"url": "https://radio-t.com/blah1",
+"site": "remark42"}}`))
+	assert.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err := client.Do(req)
+	assert.NoError(t, err)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
+	parentComment := store.Comment{}
+	require.NoError(t, render.DecodeJSON(strings.NewReader(string(body)), &parentComment))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 5)
+	require.Equal(t, 1, len(mockDestination.Get()))
+	assert.Equal(t, "", mockDestination.Get()[0].Email)
+
+	// create child comment from another user, no email notification expected
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
+		`{"text": "test 456",
+	"pid": "%s",
+	"user": {"name": "other_user"},
+	"locator":{"url": "https://radio-t.com/blah1",
+	"site": "remark42"}}`, parentComment.ID)))
+	assert.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 5)
+	require.Equal(t, 2, len(mockDestination.Get()))
+	assert.Empty(t, mockDestination.Get()[1].Email)
+
+	// send confirmation token for email
+	req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", nil)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 5)
+	require.Equal(t, 3, len(mockDestination.Get()))
+	require.NotEmpty(t, mockDestination.Get()[2].Verification)
+	verificationToken := mockDestination.Get()[2].Verification.Token
+
+	// verify email
+	req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), nil)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+
+	// get user information to verify the subscription
+	req, err = http.NewRequest(http.MethodGet, ts.URL+"/api/v1/user?site=remark42", nil)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var user store.User
+	err = json.Unmarshal(body, &user)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev", EmailSubscription: true,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
+
+	// create child comment from another user, email notification expected
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
+		`{"text": "test 789",
+	"pid": "%s",
+	"user": {"name": "other_user"},
+	"locator":{"url": "https://radio-t.com/blah1",
+	"site": "remark42"}}`, parentComment.ID)))
+	assert.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 5)
+	require.Equal(t, 4, len(mockDestination.Get()))
+	assert.Equal(t, "good@example.com", mockDestination.Get()[3].Email)
+
+	// delete user's email
+	req, err = http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/email?site=remark42", nil)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+
+	// create child comment from another user, no email notification expected
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
+		`{"text": "test 321",
+	"user": {"name": "other_user"},
+	"locator":{"url": "https://radio-t.com/blah1",
+	"site": "remark42"}}`))
+	assert.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 5)
+	require.Equal(t, 5, len(mockDestination.Get()))
+	assert.Empty(t, mockDestination.Get()[4].Email)
+}
+
 func TestRest_UserAllData(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
 	// write 3 comments
 	user := store.User{ID: "dev", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
+	require.NoError(t, err, "%+v", err)
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
@@ -456,10 +739,10 @@ func TestRest_UserAllData(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
+	strUungzBody := string(ungzBody)
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
+	assert.True(t, strings.HasPrefix(strUungzBody,
-	assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
+		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
-	t.Logf("%s", string(ungzBody))
+	assert.Equal(t, 3, strings.Count(strUungzBody, `"text":`), "3 comments inside")
 
 	parsed := struct {
 		Info     store.User      `json:"info"`
@@ -467,15 +750,15 @@ func TestRest_UserAllData(t *testing.T) {
 	}{}
 
 	err = json.Unmarshal(ungzBody, &parsed)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
-		Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, parsed.Info)
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, parsed.Info)
 	assert.Equal(t, 3, len(parsed.Comments))
 
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 401, resp.StatusCode)
 }
 
@@ -484,21 +767,21 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	defer teardown()
 
 	user := store.User{ID: "dev", Name: "user name 1"}
-	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
 
 	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
 		c.Timestamp = c.Timestamp.Add(time.Second)
 		_, err := srv.DataService.Create(c)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
@@ -506,9 +789,10 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
+	strUngzBody := string(ungzBody)
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
+	assert.True(t, strings.HasPrefix(strUngzBody,
-	assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside")
+		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+	assert.Equal(t, 51, strings.Count(strUngzBody, `"text":`), "51 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
@@ -516,31 +800,31 @@ func TestRest_DeleteMe(t *testing.T) {
 	defer teardown()
 
 	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	m := map[string]string{}
 	err = json.Unmarshal(body, &m)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, "radio-t", m["site"])
+	assert.Equal(t, "remark42", m["site"])
 	assert.Equal(t, "dev", m["user_id"])
 
 	token := m["token"]
 	claims, err := srv.Authenticator.TokenService().Parse(token)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "dev", claims.User.ID)
 	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"])
 
-	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
+	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
 
@@ -565,10 +849,10 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 		req.Header.Add("Content-Type", contentType)
 		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, 200, resp.StatusCode)
 		body, err := ioutil.ReadAll(resp.Body)
-		require.Nil(t, err)
+		require.NoError(t, err)
 
 		m := map[string]string{}
 		err = json.Unmarshal(body, &m)
@@ -582,21 +866,21 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 1462, len(body))
-	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.gif")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.jpg")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
 
 	id = savePic("picture.blah")
 	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
@@ -629,7 +913,6 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	svc.ImageService = imageService
 
 	dataService := svc.DataService
-	dataService.EditDuration = time.Millisecond * 100
 	dataService.ImageService = svc.ImageService
 	svc.privRest.dataService = dataService
 
@@ -648,40 +931,41 @@ func TestRest_CreateWithPictures(t *testing.T) {
 		req.Header.Add("Content-Type", contentType)
 		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, 200, resp.StatusCode)
 
 		body, err := ioutil.ReadAll(resp.Body)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		m := map[string]string{}
 		err = json.Unmarshal(body, &m)
 		assert.NoError(t, err)
-		assert.Contains(t, m["id"], ".png")
-		t.Logf(string(body))
 		return m["id"]
 	}
 
-	id1 := uploadPicture("pic1.png")
+	var ids [3]string
-	id2 := uploadPicture("pic2.png")
-	id3 := uploadPicture("pic3.png")
 
-	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	for i := range ids {
-	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+		ids[i] = uploadPicture(fmt.Sprintf("pic%d.png", i))
+	}
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, ids[0], ids[1], ids[2])
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`, text)
 
 	resp, err := post(t, ts.URL+"/api/v1/comment", body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
 
-	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	for i := range ids {
-	assert.NotNil(t, err, "not moved from staging yet")
+		_, err = os.Stat("/tmp/remark42/images/" + ids[i])
+		assert.Error(t, err, "picture %d not moved from staging yet", i)
+	}
 
-	time.Sleep(300 * time.Millisecond)
+	time.Sleep(1500 * time.Millisecond)
-	_, err = os.Stat("/tmp/remark42/images/" + id1)
+
-	assert.NoError(t, err, "moved from staging")
+	for i := range ids {
-	_, err = os.Stat("/tmp/remark42/images/" + id2)
+		_, err = os.Stat("/tmp/remark42/images/" + ids[i])
-	assert.NoError(t, err, "moved from staging")
+		assert.NoError(t, err, "picture %d moved from staging and available in permanent location", i)
-	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	}
-	assert.NoError(t, err, "moved from staging")
 }

backend/app/rest/api/rest_public.go

@@ -13,9 +13,9 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
@@ -26,7 +26,7 @@ import (
 
 type public struct {
 	dataService      pubStore
-	cache            cache.LoadingCache
+	cache            LoadingCache
 	readOnlyAge      int
 	commentFormatter *store.CommentFormatter
 	imageService     *image.Service

backend/app/rest/api/rest_public_test.go

@@ -12,9 +12,8 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
+	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -36,14 +35,14 @@ func TestRest_Preview(t *testing.T) {
 	defer teardown()
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
 
 	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
@@ -64,13 +63,12 @@ BKT
 	text = strings.Replace(text, "BKT", "```", -1)
 	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
 	j = strings.Replace(j, "\n", "\\n", -1)
-	t.Log(j)
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
 }
 
@@ -78,30 +76,30 @@ func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id1 := addComment(t, c1, ts)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id2 := addComment(t, c2, ts)
 
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
 	assert.Equal(t, "<p>test test #1</p>\n", comments.Comments[0].Text)
@@ -112,21 +110,21 @@ func TestRest_Find(t *testing.T) {
 	assert.True(t, comments.Info.FirstTS.Before(comments.Info.LastTS))
 
 	// get sorted by -time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=-time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=-time")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[1].ID)
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
 	tree := service.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 1, len(tree.Nodes))
+	require.Equal(t, 1, len(tree.Nodes))
 	assert.Equal(t, 1, len(tree.Nodes[0].Replies))
 	assert.Equal(t, 2, tree.Info.Count)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
@@ -138,28 +136,28 @@ func TestRest_FindAge(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	tree := service.Tree{}
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is fresh")
 
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is old")
 }
@@ -169,38 +167,38 @@ func TestRest_FindReadOnly(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
 
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -2),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	// set post to read-only
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	tree := service.Tree{}
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
 	tree = service.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is writable")
 }
@@ -209,30 +207,30 @@ func TestRest_FindUserView(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&view=user")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&view=user")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id1 := addComment(t, c1, ts)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id2 := addComment(t, c2, ts)
 
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time with view=user
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time&view=user")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time&view=user")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
 	assert.Equal(t, "dev", comments.Comments[0].User.ID)
@@ -245,14 +243,14 @@ func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "[]\n", res, "empty last should return empty list")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
 	ts1 := time.Now().UnixNano() / 1000000
@@ -262,55 +260,54 @@ func TestRest_Last(t *testing.T) {
 	ts2 := time.Now().UnixNano() / 1000000
 	id2 := addComment(t, c2, ts)
 
-	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments), "should have 2 comments")
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
-	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts1))
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts1))
 	assert.Equal(t, 200, code)
 	comments = []store.Comment{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	require.Equal(t, 2, len(comments), "should have 2 comments")
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
-	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts2))
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts2))
 	assert.Equal(t, 200, code)
 	comments = []store.Comment{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 1, len(comments), "should have 1 comments")
+	require.Equal(t, 1, len(comments), "should have 1 comments")
 	assert.Equal(t, id2, comments[0].ID)
 
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	res, code = get(t, ts.URL+"/api/v1/last/X?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/X?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
+	err = srv.DataService.Delete(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	srv.Cache.Flush(cache.FlusherRequest{})
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
-	t.Logf("%+v", comments)
 
-	_, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t-BLAH")
+	_, code = get(t, ts.URL+"/api/v1/last/2?site=remark42-BLAH")
 	assert.Equal(t, 500, code)
 }
 
@@ -319,9 +316,9 @@ func TestRest_FindUserComments(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
 	addComment(t, c1, ts)
@@ -333,10 +330,10 @@ func TestRest_FindUserComments(t *testing.T) {
 	err := srv.DataService.Delete(c2.Locator, id, store.SoftDelete)
 	assert.NoError(t, err)
 
-	_, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=blah")
+	_, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=blah")
 	assert.Equal(t, 400, code, "noting for user blah")
 
-	res, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=dev")
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
 	assert.Equal(t, 200, code)
 
 	resp := struct {
@@ -345,21 +342,26 @@ func TestRest_FindUserComments(t *testing.T) {
 	}{}
 
 	err = json.Unmarshal([]byte(res), &resp)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, 3, len(resp.Comments), "should have 3 comments")
+	require.Equal(t, 3, len(resp.Comments), "should have 3 comments")
 	assert.Equal(t, 4, resp.Count, "should have 3 count")
+
+	// user comment sorted with -time
+	assert.True(t, resp.Comments[0].Timestamp.After(resp.Comments[1].Timestamp))
+	assert.True(t, resp.Comments[1].Timestamp.After(resp.Comments[2].Timestamp))
 }
 
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=radio-t")
+	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=remark42")
 	assert.Equal(t, 200, code)
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, user)
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 
 func TestRest_Count(t *testing.T) {
@@ -367,9 +369,9 @@ func TestRest_Count(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -377,20 +379,20 @@ func TestRest_Count(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3.0, j["count"])
 
-	body, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah2")
+	body, code = get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah2")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2.0, j["count"])
 
-	_, code = get(t, ts.URL+"/api/v1/count?site=radio-t-BLAH&url=https://radio-t.com/blah1XXX")
+	_, code = get(t, ts.URL+"/api/v1/count?site=remark42-BLAH&url=https://radio-t.com/blah1XXX")
 	assert.Equal(t, 400, code)
 }
 
@@ -399,9 +401,9 @@ func TestRest_Counts(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -409,16 +411,16 @@ func TestRest_Counts(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	j := []store.PostInfo{}
 	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
 		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
 
@@ -432,9 +434,9 @@ func TestRest_List(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -442,17 +444,17 @@ func TestRest_List(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
 	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
 	assert.Equal(t, 3, pi[1].Count)
 
-	_, code = get(t, ts.URL+"/api/v1/list?site=radio-t-BLAH")
+	_, code = get(t, ts.URL+"/api/v1/list?site=remark42-BLAH")
 	assert.Equal(t, 400, code)
 }
 
@@ -461,11 +463,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah3"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -475,11 +477,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	addComment(t, c3, ts)
 	addComment(t, c3, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42&skip=1&limit=2")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	require.Equal(t, 2, len(pi))
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
@@ -491,11 +493,11 @@ func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/config?site=remark42")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 300., j["edit_duration"])
 	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
 	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
@@ -506,7 +508,6 @@ func TestRest_Config(t *testing.T) {
 	assert.Equal(t, 10., j["readonly_age"])
 	assert.Equal(t, 10000., j["max_image_size"])
 	assert.Equal(t, true, j["emoji_enabled"].(bool))
-	t.Logf("%+v", j)
 }
 
 func TestRest_Info(t *testing.T) {
@@ -516,33 +517,33 @@ func TestRest_Info(t *testing.T) {
 	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 
 	user := store.User{ID: "user1", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
+	require.NoError(t, err, "%+v", err)
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	body, code := get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 
 	info := store.PostInfo{}
 	err = json.Unmarshal([]byte(body), &info)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
 		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 	assert.Equal(t, exp, info)
 
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42-no&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
 }
 
@@ -551,32 +552,30 @@ func TestRest_InfoStream(t *testing.T) {
 	defer teardown()
 	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 800 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
 	postComment(t, ts.URL)
 
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 0; i < 10; i++ {
 			time.Sleep(10 * time.Millisecond)
 			postComment(t, ts.URL)
 		}
 	}()
 
-	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
-	wg.Wait()
+	<-done
 
-	t.Logf(string(body))
 	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
 	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
 	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
 	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
 
-	_, code = get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah123")
+	_, code = get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah123")
 	assert.Equal(t, 500, code)
 }
 
@@ -595,7 +594,7 @@ func TestRest_InfoStreamTooMany(t *testing.T) {
 	wg.Add(20)
 	for i := 0; i < 20; i++ {
 		go func() {
-			_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+			_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
 			if code == 429 {
 				atomic.AddInt32(&errsCount, 1)
 			}
@@ -617,7 +616,7 @@ func TestRest_InfoStreamTimeout(t *testing.T) {
 	postComment(t, ts.URL)
 
 	st := time.Now()
-	_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
 }
@@ -627,43 +626,38 @@ func TestRest_InfoStreamCancel(t *testing.T) {
 	defer teardown()
 	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 1500 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
 	postComment(t, ts.URL)
 
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 0; i < 5; i++ {
-			time.Sleep(100 * time.Millisecond)
+			time.Sleep(300 * time.Millisecond)
 			postComment(t, ts.URL)
-			log.Printf("write #%d", i)
 		}
 	}()
 
 	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
 	defer cancel()
 	req = req.WithContext(ctx)
-	log.Print("start req")
 	r, err := client.Do(req)
-	log.Print("end req")
+	require.NoError(t, err)
-	require.Nil(t, err)
 	defer r.Body.Close()
+	<-ctx.Done()
+	<-done
 	body, err := ioutil.ReadAll(r.Body)
 	require.EqualError(t, err, "context deadline exceeded")
 	assert.Equal(t, 200, r.StatusCode)
 
-	wg.Wait()
+	recs := strings.Count(string(body), "data:")
-
+	require.Equal(t, 1, recs, "should have 1 event:\n", string(body))
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	assert.Contains(t, string(body), `"count":2`)
-	require.Equal(t, 2*3, len(recs), "should have 2 events")
-	assert.True(t, strings.Contains(recs[0*3+1], `"count":2`), recs[0])
-	assert.True(t, strings.Contains(recs[1*3+1], `"count":3`), recs[1])
 }
 
 func TestRest_InfoStreamSince(t *testing.T) {
@@ -671,27 +665,24 @@ func TestRest_InfoStreamSince(t *testing.T) {
 	defer teardown()
 	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
-	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 900 * time.Millisecond
 	srv.pubRest.streamer.MaxActive = 100
 
 	postComment(t, ts.URL)
 
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 0; i < 10; i++ {
-			time.Sleep(10 * time.Millisecond)
+			time.Sleep(15 * time.Millisecond)
 			postComment(t, ts.URL)
 		}
 	}()
 
-	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1&since=12345678")
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1&since=12345678")
 	assert.Equal(t, 200, code)
-	wg.Wait()
+	<-done
-
+	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
-	t.Logf(string(body))
-	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
 	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
 }
 
@@ -716,10 +707,9 @@ func TestRest_LastCommentsStream(t *testing.T) {
 	postComment(t, ts.URL)
 
 	defer teardown()
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 1; i < 10; i++ {
 			time.Sleep(100 * time.Millisecond)
 			postComment(t, ts.URL)
@@ -727,23 +717,21 @@ func TestRest_LastCommentsStream(t *testing.T) {
 	}()
 
 	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
+	<-done
 	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, r.StatusCode)
 
-	wg.Wait()
-	t.Logf("headers: %+v", r.Header)
 	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
 	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
 
 	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
 	require.Equal(t, 9*3, len(recs), "9 events")
-	t.Logf("%s", string(body))
 	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
 }
 
@@ -758,7 +746,7 @@ func TestRest_LastCommentsStreamTimeout(t *testing.T) {
 	postComment(t, ts.URL)
 
 	st := time.Now()
-	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
 	assert.Equal(t, 200, code)
 	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
 }
@@ -773,10 +761,9 @@ func TestRest_LastCommentsStreamCancel(t *testing.T) {
 	postComment(t, ts.URL)
 
 	defer teardown()
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 1; i < 10; i++ {
 			time.Sleep(100 * time.Millisecond)
 			postComment(t, ts.URL)
@@ -784,23 +771,21 @@ func TestRest_LastCommentsStreamCancel(t *testing.T) {
 	}()
 
 	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
 	defer cancel()
 	req = req.WithContext(ctx)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	<-done
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
 	require.EqualError(t, err, "context deadline exceeded")
 	assert.Equal(t, 200, r.StatusCode)
 
-	wg.Wait()
-
 	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	require.Equal(t, 2*3, len(recs), "2 events")
+	assert.True(t, len(recs) < 30, "less 10 events")
-	assert.True(t, strings.Contains(recs[0+1], `test 123`), recs[0+1])
 }
 
 func TestRest_LastCommentsStreamTooMany(t *testing.T) {
@@ -818,7 +803,7 @@ func TestRest_LastCommentsStreamTooMany(t *testing.T) {
 	wg.Add(20)
 	for i := 0; i < 20; i++ {
 		go func() {
-			_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+			_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
 			if code == 429 {
 				atomic.AddInt32(&errsCount, 1)
 			}
@@ -828,7 +813,7 @@ func TestRest_LastCommentsStreamTooMany(t *testing.T) {
 	wg.Wait()
 	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
 
-	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
 	assert.Equal(t, 200, code, "all streams closed, good to go again")
 }
 
@@ -842,40 +827,37 @@ func TestRest_LastCommentsStreamSince(t *testing.T) {
 	postComment(t, ts.URL)
 
 	defer teardown()
-	wg := sync.WaitGroup{}
+	done := make(chan struct{})
-	wg.Add(1)
 	go func() {
-		defer wg.Done()
+		defer close(done)
 		for i := 1; i < 10; i++ {
-			time.Sleep(100 * time.Millisecond)
+			time.Sleep(50 * time.Millisecond)
 			postComment(t, ts.URL)
 		}
 	}()
 
 	client := http.Client{}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t&since=123456", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42&since=123456", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	<-done
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, r.StatusCode)
 
-	wg.Wait()
-	t.Logf("headers: %+v", r.Header)
 	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
 
 	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
-	require.Equal(t, 10*3, len(recs), "10 events, includes first record")
+	require.Equal(t, 10*3, len(recs), "should be 10 events, including first record:\n", recs)
-	t.Logf("%v", recs)
 }
 
 func postComment(t *testing.T, url string) {
-	resp, e := post(t, url+"/api/v1/comment",
+	resp, err := post(t, url+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
-	require.Nil(t, e)
+	require.NoError(t, err)
-	b, e := ioutil.ReadAll(resp.Body)
+	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, e)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
 }

backend/app/rest/api/rest_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -20,13 +21,14 @@ import (
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
@@ -36,35 +38,36 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testHTML = "/tmp/test-remark.html"
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
-var getStartedHTML = "/tmp/getstarted.html"
 
-var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." +
+var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
-	"eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIs" +
-	"InVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMT" +
-	"I3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
-var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9." +
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
-	"eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1" +
+
-	"NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0" +
+var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
-	"dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlI" +
-	"iwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.gAR_sZT7hTx7CNHByyrJQWMB5tAtoiISAiG8kes1IjA"
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/web/test-remark.html")
+	testHtmlName := "test-remark.html"
+	testHTMLFile := os.TempDir() + "/" + testHtmlName
+	err := ioutil.WriteFile(testHTMLFile, []byte("some html"), 0700)
+	assert.NoError(t, err)
+
+	body, code := get(t, ts.URL+"/web/"+testHtmlName)
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "some html", body)
+	_ = os.Remove(testHTMLFile)
 }
 
 func TestRest_GetStarted(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
+	getStartedHTML := os.TempDir() + "/getstarted.html"
 	err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	body, code := get(t, ts.URL+"/index.html")
 	assert.Equal(t, 200, code)
@@ -78,15 +81,20 @@ func TestRest_GetStarted(t *testing.T) {
 
 func TestRest_Shutdown(t *testing.T) {
 	srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
+	done := make(chan bool)
 
+	// without waiting for channel close at the end goroutine will stay alive after test finish
+	// which would create data race with next test
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(200 * time.Millisecond)
 		srv.Shutdown()
+		close(done)
 	}()
 
 	st := time.Now()
 	srv.Run(0)
 	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
+	<-done
 }
 
 func TestRest_filterComments(t *testing.T) {
@@ -105,6 +113,7 @@ func TestRest_filterComments(t *testing.T) {
 }
 
 func TestRest_RunStaticSSLMode(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: auth.NewService(auth.Opts{
 			AvatarStore:       avatar.NewLocalFS("/tmp"),
@@ -114,18 +123,19 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		ImageProxy: &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Static,
-			Port:    8443,
+			Port:    sslPort,
 			Key:     "../../cmd/testdata/key.pem",
 			Cert:    "../../cmd/testdata/cert.pem",
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
-		srv.Run(38080)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -139,40 +149,42 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38080/blah?param=1")
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
-	resp, err = client.Get("https://localhost:8443/ping")
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	srv.Shutdown()
 }
 
 func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: &auth.Service{},
 		ImageProxy:    &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Auto,
-			Port:    8443,
+			Port:    sslPort,
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
 		// can't check https server locally, just only http server
-		srv.Run(38081)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -181,11 +193,11 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38081/blah?param=1")
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	srv.Shutdown()
 }
@@ -281,23 +293,63 @@ func TestRest_parseError(t *testing.T) {
 	}
 }
 
+func TestRest_cacheControl(t *testing.T) {
+
+	tbl := []struct {
+		url     string
+		version string
+		exp     time.Duration
+		etag    string
+		maxAge  int
+	}{
+		{"http://example.com/foo", "v1", time.Hour, "b433be1ea19edaee9dc92ca4b895b6bdf3c058cb", 3600},
+		{"http://example.com/foo2", "v1", 10 * time.Hour, "6d8466aef3246c1057452561acddf7ad9d0d99e0", 36000},
+		{"http://example.com/foo", "v2", time.Hour, "481700c52aab0dfbca99f3ffc2a4fbb27884c114", 3600},
+		{"https://example.com/foo", "v2", time.Hour, "bebd4f1b87f474792c4e75e5affe31fbf67f5778", 3600},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := httptest.NewRequest("GET", tt.url, nil)
+			w := httptest.NewRecorder()
+
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h.ServeHTTP(w, req)
+			resp := w.Result()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			t.Logf("%+v", resp.Header)
+			assert.Equal(t, `"`+tt.etag+`"`, resp.Header.Get("Etag"))
+			assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds())), resp.Header.Get("Cache-Control"))
+
+		})
+	}
+
+}
+
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
-	// log.Setup(log.Out(ioutil.Discard))
 	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 
-	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
+	tmp := os.TempDir()
-	os.Remove(testDb)
+	var testDb string
-	os.Remove(testHTML)
+	// pick a file name which is not in use for sure
-	os.RemoveAll("/tmp/ava-remark42")
+	for i := 0; i < 10; i++ {
-	os.RemoveAll("/tmp/pics-remark42")
+		testDb = fmt.Sprintf("/%s/test-remark-%d.db", tmp, rand.Int31())
+		_, err := os.Stat(testDb)
+		if err != nil {
+			break
+		}
+	}
+	_ = os.RemoveAll(tmp + "/ava-remark42")
+	_ = os.RemoveAll(tmp + "/pics-remark42")
 
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	memCache, err := cache.NewMemoryCache()
+	cacheBackend, err := cache.NewExpirableCache()
-	assert.NoError(t, err)
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
 
-	astore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
@@ -314,17 +366,17 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
 			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
-			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
+			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
-		WebRoot:   "/tmp",
+		WebRoot:   tmp,
 		RemarkURL: "https://demo.remark42.com",
 		ImageService: &image.Service{
 			Store: &image.FileSystem{
-				Location:   "/tmp/pics-remark42",
+				Location:   tmp + "/pics-remark42",
 				Partitions: 100,
 				MaxSize:    10000,
-				Staging:    "/tmp/pics-remark42/staging",
+				Staging:    tmp + "/pics-remark42/staging",
 			},
 			TTL: time.Millisecond * 100,
 		},
@@ -336,7 +388,8 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
-			Cache:             &cache.Nop{},
+			UrlMapperMaker:    migrator.NewUrlMapper,
+			Cache:             memCache,
 			KeyStore:          astore,
 		},
 		Streamer: &Streamer{
@@ -344,28 +397,25 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			TimeOut:   5 * time.Second,
 			MaxActive: 100,
 		},
-		EmojiEnabled: true,
+		NotifyService: notify.NopService,
+		EmojiEnabled:  true,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-
 	ts = httptest.NewServer(srv.routes())
 
 	teardown = func() {
 		ts.Close()
 		require.NoError(t, srv.DataService.Close())
-		os.Remove(testDb)
+		_ = os.Remove(testDb)
-		os.Remove(testHTML)
+		_ = os.RemoveAll(tmp + "/ava-remark42")
-		os.RemoveAll("/tmp/ava-remark42")
+		_ = os.RemoveAll(tmp + "/pics-remark42")
-		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
-// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+// fake auth middleware make user authenticated and uses query's fake_id for ID and fake_name for Name
 func fakeAuth(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Query().Get("fake_id") != "" {
@@ -381,10 +431,10 @@ func fakeAuth(next http.Handler) http.Handler {
 
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	return string(body), r.StatusCode
 }
 
@@ -399,53 +449,53 @@ func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	b, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(b), r.StatusCode
 }
 
 func getWithAdminAuth(t *testing.T, url string) (string, int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(body), r.StatusCode
 }
 func post(t *testing.T, url string, body string) (*http.Response, error) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	return client.Do(req)
 }
 
 func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	b, err := json.Marshal(c)
-	require.Nil(t, err, "can't marshal comment %+v", c)
+	require.NoError(t, err, "can't marshal comment %+v", c)
 
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	crResp := R.JSON{}
 	err = json.Unmarshal(b, &crResp)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
@@ -459,3 +509,26 @@ func requireAdminOnly(t *testing.T, req *http.Request) {
 	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
 }
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}

backend/app/rest/api/rss.go

@@ -5,8 +5,8 @@ import (
 	"net/http"
 	"time"
 
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
 	"github.com/pkg/errors"
 
@@ -16,7 +16,7 @@ import (
 
 type rss struct {
 	dataService rssStore
-	cache       cache.LoadingCache
+	cache       LoadingCache
 }
 
 type rssStore interface {

backend/app/rest/api/rss_test.go

@@ -22,7 +22,7 @@ func TestServer_RssPost(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "1234567890",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	id1, err := rst.DataService.Create(c1)
@@ -30,7 +30,7 @@ func TestServer_RssPost(t *testing.T) {
 	assert.Equal(t, "1234567890", id1)
 	pubDate := time.Now().Format(time.RFC1123Z)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -54,7 +54,7 @@ func TestServer_RssPost(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
+	_, code = get(t, ts.URL+"/api/v1/rss/post?site=remark42-bad&url=https://radio-t.com/blah1")
 	assert.Equal(t, 400, code)
 }
 
@@ -69,13 +69,13 @@ func TestServer_RssSite(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:      "comment-id-2",
 		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -85,15 +85,15 @@ func TestServer_RssSite(t *testing.T) {
 	require.NoError(t, err)
 
 	require.NoError(t, err)
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/site?site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 		  <channel>
 		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
+		    <link>remark42</link>
-		    <description>site comment for radio-t</description>
+		    <description>site comment for remark42</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -132,14 +132,14 @@ func TestServer_RssWithReply(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-id-2",
 		ParentID: "comment-id-1",
 		Text:     "xyz test",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:     store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -148,7 +148,7 @@ func TestServer_RssWithReply(t *testing.T) {
 	_, err = rst.DataService.Create(c2)
 	require.NoError(t, err)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah10")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -192,34 +192,34 @@ func TestServer_RssReplies(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-1",
 		Text:    "c1",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "user1", Name: "user1"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-2",
 		Text:     "reply to c1 from user2",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user2", Name: "user2"},
 	}
 	c3 := store.Comment{
 		ID:       "comment-3",
 		Text:     "reply to c1 from user3",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user3", Name: "user3"},
 	}
 	c4 := store.Comment{
 		ID:       "comment-4",
 		Text:     "reply to c2 from developer one",
 		ParentID: "comment-2",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "dev", Name: "developer one"},
 	}
 	c5 := store.Comment{
 		ID:      "comment-5",
 		Text:    "developer one",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "dev", Name: "developer one"},
 	}
 
@@ -235,13 +235,13 @@ func TestServer_RssReplies(t *testing.T) {
 	require.NoError(t, err)
 
 	// replies to c1 (user1). Must be [c3, c2]
-	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 	    <channel>
 	        <title>Remark42 comments</title>
-	        <link>radio-t</link>
+	        <link>remark42</link>
 	        <description>replies to user1</description>
 	        <pubDate>%s</pubDate>
 	        <item>
@@ -265,7 +265,7 @@ func TestServer_RssReplies(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t-bad")
+	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42-bad")
 	assert.Equal(t, 400, code)
 }
 

backend/app/rest/api/ssl.go

@@ -119,7 +119,7 @@ func (s *Rest) makeTLSConfig() *tls.Config {
 			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			// tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
 		},
 		MinVersion: tls.VersionTLS12,
 		CurvePreferences: []tls.CurveID{

backend/app/rest/api/ssl_test.go

@@ -33,7 +33,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
@@ -62,28 +62,28 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
 
 	// check acme http challenge
 	req, err := http.NewRequest("GET", ts.URL+"/.well-known/acme-challenge/token123", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Host = "localhost" // for passing hostPolicy check
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 404, resp.StatusCode)
 
 	err = m.Cache.Put(context.Background(), "token123+http-01", []byte("token"))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "token", string(body))
 }

backend/app/rest/httperrors.go

@@ -1,7 +1,10 @@
 package rest
 
 import (
+	"bytes"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"net/url"
 	"runtime"
@@ -35,9 +38,51 @@ const (
 	ErrAssetNotFound      = 18 // requested file not found
 )
 
+const errorHtml = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">{{.Error}}: {{.Details}}.</p>
+</div>
+</body>
+</html>
+`
+
+// errTmplData store data for error message
+type errTmplData struct {
+	Error   string
+	Details string
+}
+
+// SendErrorHTML makes html body with provided template and responds with provided http status code,
+// error code is not included in render as it is intended for UI developers and not for the users
+func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err := tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("error").Parse(errorHtml))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	render.Status(r, httpStatusCode)
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, errTmplData{
+		Error:   err.Error(),
+		Details: details,
+	})
+	render.HTML(w, r, msg.String())
+}
+
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
 func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
 	render.Status(r, httpStatusCode)
 	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
@@ -56,14 +101,10 @@ func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details strin
 	if pc, file, line, ok := runtime.Caller(2); ok {
 		fnameElems := strings.Split(file, "/")
 		funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-		srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
+		srcFileInfo = fmt.Sprintf("[%s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
 			line, funcNameElems[len(funcNameElems)-1])
 	}
 
-	remoteIP := r.RemoteAddr
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s",
-	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
+		details, err, httpStatusCode, errCode, uinfoStr, q, srcFileInfo)
-		remoteIP = remoteIP[:pos]
-	}
-	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
-		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -26,23 +27,51 @@ func TestSendErrorJSON(t *testing.T) {
 	defer ts.Close()
 
 	resp, err := http.Get(ts.URL + "/error")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
 	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
+func TestSendErrorHTML(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/error" {
+			t.Log("http err request", r.URL)
+			SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL + "/error")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 500, resp.StatusCode)
+
+	assert.NotContains(t, string(body), `987`, "user html should not contain internal error code")
+	assert.Contains(t, string(body), `error details 123456`)
+	assert.Contains(t, string(body), `error 500`)
+}
+
 func TestErrorDetailsMsg(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		req.RemoteAddr = "1.2.3.4"
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (123) - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsg]")
 	}
 	callerFn()
 }
@@ -53,10 +82,11 @@ func TestErrorDetailsMsgWithUser(t *testing.T) {
 		require.NoError(t, err)
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
-		require.Nil(t, err)
+		require.NoError(t, err)
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example." +
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (34567) - test/id - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
-			"com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:61 rest.TestErrorDetailsMsgWithUser]", msg)
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsgWithUser]")
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,10 +1,15 @@
 package proxy
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
@@ -14,36 +19,77 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // Image extracts image src from comment's html and provides proxy for them
 // this is needed to keep remark42 running behind of HTTPS serve all images via https
 type Image struct {
-	RemarkURL string
+	RemarkURL     string
-	RoutePath string
+	RoutePath     string
-	Enabled   bool
+	HTTP2HTTPS    bool
-	Timeout   time.Duration
+	CacheExternal bool
+	Timeout       time.Duration
+	ImageService  *image.Service
 }
 
-// Convert all img src links without https to proxied links
+// Convert img src links to proxied links depends on enabled options
 func (p Image) Convert(commentHTML string) string {
-	if !p.Enabled || strings.HasPrefix(p.RemarkURL, "http://") {
+	if p.CacheExternal {
-		return commentHTML
+		imgs, err := p.extract(commentHTML, func(img string) bool { return !strings.HasPrefix(img, p.RemarkURL) })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
 	}
 
-	imgs, err := p.extract(commentHTML)
+	if p.HTTP2HTTPS && !strings.HasPrefix(p.RemarkURL, "http://") {
+		imgs, err := p.extract(commentHTML, func(img string) bool { return strings.HasPrefix(img, "http://") })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
+	}
+
+	return commentHTML
+}
+
+// extract gets all images matching predicate and return list of src
+func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]string, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
-		return commentHTML
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+		if im, ok := s.Attr("src"); ok {
+			if imgSrcPred(im) {
+				result = append(result, im)
+			}
+		}
+	})
+	return result, nil
+}
+
+// replace img links in commentHTML with route to proxy, base64 encoded original link
+func (p Image) replace(commentHTML string, imgs []string) string {
+	for _, img := range imgs {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
+		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
+		commentHTML = strings.Replace(commentHTML, img, resImgURL, -1)
 	}
 
-	return p.replace(commentHTML, imgs)
+	return commentHTML
 }
 
 // Handler returns http handler respond to proxied request
 func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
-
+	if !p.HTTP2HTTPS && !p.CacheExternal {
-	if !p.Enabled {
+		// TODO: we might need to find a better way to handle it. If admin enables caching/proxy and disables it later on
-		http.Error(w, "proxy disabled", http.StatusNotImplemented)
+		// all comments that got converted will lose their images. We can't just return a redirect (it will open an ability
+		// to redirect anywhere). We can probably continue proxying these images (but need to make sure this behavior is
+		// documented) or, better, provide a way to migrate back converted comments.
+		http.Error(w, "none of the proxy features are enabled", http.StatusNotImplemented)
 		return
 	}
 
@@ -53,48 +99,42 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	timeout := 60 * time.Second // default
+	imgURL := string(src)
-	if p.Timeout > 0 {
+	var imgReader io.ReadCloser
-		timeout = p.Timeout
+	imgID, err := cachedImgID(imgURL)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	client := http.Client{Timeout: 30 * time.Second}
-	var resp *http.Response
-	err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-		var e error
-		req, e := http.NewRequest("GET", string(src), nil)
-		if e != nil {
-			return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-		}
-		resp, e = client.Do(req.WithContext(ctx))
-		return e
-	})
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse image url "+imgURL, rest.ErrAssetNotFound)
 		return
 	}
+	if p.CacheExternal {
+		imgReader, _, err = p.ImageService.Load(imgID)
+		if err != nil {
+			imgReader = nil
+		}
+	}
+	if imgReader == nil {
+		imgReader, err = p.downloadImage(context.Background(), imgURL)
+		if err != nil {
+			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
+			return
+		}
+		if p.CacheExternal {
+			var buf bytes.Buffer
+			// We need to duplicate data into a new buffer because `cacheImage` would read provider Reader
+			// and we would need another one to read data for response
+			p.cacheImage(io.TeeReader(imgReader, &buf), imgID)
+			if err := imgReader.Close(); err != nil {
+				log.Printf("[WARN] can't close image reader, %s", err)
+			}
+			imgReader = ioutil.NopCloser(&buf)
+		}
+	}
 	defer func() {
-		if e := resp.Body.Close(); e != nil {
+		if e := imgReader.Close(); e != nil {
-			log.Printf("[WARN] can't close body, %s", e)
+			log.Printf("[WARN] can't close image reader, %s", e)
 		}
 	}()
 
-	if resp.StatusCode != http.StatusOK {
-		w.WriteHeader(resp.StatusCode)
-		return
-	}
-
-	for k, v := range resp.Header {
-		if strings.EqualFold(k, "Content-Type") {
-			w.Header().Set(k, v[0])
-		}
-		if strings.EqualFold(k, "Content-Length") {
-			w.Header().Set(k, v[0])
-		}
-	}
 	// enforce client-side caching
 	etag := `"` + r.URL.Query().Get("src") + `"`
 	w.Header().Set("Etag", etag)
@@ -105,36 +145,78 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	if _, e := io.Copy(w, resp.Body); e != nil {
-		log.Printf("[WARN] can't copy image stream, %s", e)
-	}
-}
 
-// extract gets all non-https images and return list of src
+	w.Header().Add("Content-Type", "image/*")
-func (p Image) extract(commentHTML string) ([]string, error) {
+	_, err = io.Copy(w, imgReader)
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
-		return nil, errors.Wrap(err, "can't create document")
+		log.Printf("[WARN] can't copy image stream, %s", err)
 	}
-	result := []string{}
+}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+
-		if im, ok := s.Attr("src"); ok {
+// cache image from provided Reader using given ID
-			if strings.HasPrefix(im, "http://") {
+func (p Image) cacheImage(r io.Reader, imgID string) {
-				result = append(result, im)
+	id, err := p.ImageService.SaveWithID(imgID, r)
-			}
+	if err != nil {
+		log.Printf("[WARN] unable to save image to the storage: %+v", err)
+	}
+	// In the future we can do something smarter than just committing everything (eg, some kind of LFU/LRU)
+	if err := p.ImageService.Commit(id); err != nil {
+		log.Printf("[WARN] unable to commit image %s", imgID)
+	}
+}
+
+// download an image. Returns a Reader which has to be closed by a caller
+func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser, error) {
+	log.Printf("[DEBUG] downloading image %s", imgURL)
+
+	timeout := 60 * time.Second // default
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	client := http.Client{Timeout: 30 * time.Second}
+	var resp *http.Response
+	err := repeater.NewDefault(5, time.Second).Do(ctx, func() error {
+		var e error
+		req, e := http.NewRequest("GET", imgURL, nil)
+		if e != nil {
+			return errors.Wrapf(e, "failed to make request for %s", imgURL)
 		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
 	})
-	return result, nil
+	if err != nil {
-}
+		log.Print(err.Error())
-
+		return nil, err
-// replace img links in commentHTML with route to proxy, base64 encoded original link
-func (p Image) replace(commentHTML string, imgs []string) string {
-
-	for _, img := range imgs {
-		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
-		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
-		commentHTML = strings.Replace(commentHTML, img, resImgURL, -1)
 	}
 
-	return commentHTML
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
+	}
+
+	imgData, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("unable to read image body")
+	}
+	return ioutil.NopCloser(bytes.NewBuffer(imgData)), nil
+}
+
+func sha1Str(s string) string {
+	return fmt.Sprintf("%x", sha1.Sum([]byte(s))) // nolint
+}
+
+// generates ID for a cached image.
+// ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
+// <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
+// <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
+//                              (as accurate as deduplication based on potentially mutable url can be)
+func cachedImgID(imgURL string) (string, error) {
+	parsedURL, err := url.Parse(imgURL)
+	if err != nil {
+		return "", errors.Wrapf(err, "can parse url %s", imgURL)
+	}
+	return fmt.Sprintf("cached_images/%s-%s", sha1Str(parsedURL.Hostname()), sha1Str(imgURL)), nil
 }

backend/app/rest/proxy/image_test.go

@@ -1,8 +1,10 @@
 package proxy
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -12,7 +14,9 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 func TestPicture_Extract(t *testing.T) {
@@ -46,26 +50,26 @@ func TestPicture_Extract(t *testing.T) {
 			[]string{},
 		},
 	}
-	img := Image{Enabled: true}
+	img := Image{HTTP2HTTPS: true}
 
 	for i, tt := range tbl {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			res, err := img.extract(tt.inp)
+			res, err := img.extract(tt.inp, func(src string) bool { return strings.HasPrefix(src, "http://") })
-			assert.Nil(t, err)
+			assert.NoError(t, err)
 			assert.Equal(t, tt.res, res)
 		})
 	}
 }
 
 func TestPicture_Replace(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
 		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 }
 
 func TestImage_Routes(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
+	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
@@ -75,25 +79,85 @@ func TestImage_Routes(t *testing.T) {
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	t.Logf("%+v", resp.Header)
 	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 404, resp.StatusCode)
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
+func TestImage_Routes_CachingImage(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  &image.Service{Store: &imageStore},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+
+	imgURL := httpSrv.URL + "/image/img1.png"
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, int64(0), nil)
+	imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Run(func(args mock.Arguments) { _, _ = ioutil.ReadAll(args.Get(1).(io.Reader)) }).Return("", nil)
+	imageStore.On("Commit", mock.Anything).Once().Return(nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "123", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+	imageStore.AssertCalled(t, "SaveWithID", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+sha1Str(imgURL), mock.Anything)
+	imageStore.AssertCalled(t, "Commit", mock.Anything)
+}
+
+func TestImage_Routes_Using_Cachded_Image(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  &image.Service{Store: &imageStore},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	// In order to validate that cached data is used cache "will return" some other data from what http server would
+	imageReader := ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("%256s", "X"))))
+	imageStore.On("Load", mock.Anything).Once().Return(imageReader, int64(256), nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "256", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+}
+
 func TestImage_RoutesTimedOut(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
 
 	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
@@ -102,31 +166,52 @@ func TestImage_RoutesTimedOut(t *testing.T) {
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	assert.Equal(t, 404, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
 }
 
-func TestPicture_Convert(t *testing.T) {
+func TestPicture_Convert_ProxyMode(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
 	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://radio-t.com/img3.png"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
-	img = Image{Enabled: true, RoutePath: "/img", RemarkURL: "http://example.com"}
+	img = Image{HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "http://example.com"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "http:// remark url, no proxy")
 
-	img = Image{Enabled: false, RoutePath: "/img"}
+	img = Image{HTTP2HTTPS: false, RoutePath: "/img"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
 }
 
+func TestPicture_Convert_CachingMode(t *testing.T) {
+	img := Image{CacheExternal: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+
+	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="https://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cHM6Ly9yYWRpby10LmNvbS9pbWczLnBuZw=="/> xyz <img src="https://remark42.com/img?src=aHR0cHM6Ly9pbWFnZXMucGV4ZWxzLmNvbS82NzYzNi9pbWc0LmpwZWc=">`, r)
+
+	r = img.Convert(`<img src="https://remark42.com/pictures/1.png"/>`)
+	assert.Equal(t, `<img src="https://remark42.com/pictures/1.png"/>`, r)
+
+	img = Image{CacheExternal: false, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/>`)
+	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/>`, r)
+
+	// both Caching and Proxy are enabled
+	img = Image{CacheExternal: true, HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+}
+
 func imgHTTPServer(t *testing.T) *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/image/img1.png" {

backend/app/rest/user.go

@@ -35,6 +35,7 @@ func GetUserInfo(r *http.Request) (user store.User, err error) {
 		Admin:    u.IsAdmin(),
 		Verified: u.BoolAttr("verified"),
 		Blocked:  u.BoolAttr("blocked"),
+		SiteID:   u.Audience,
 	}, nil
 }
 
@@ -50,10 +51,11 @@ func GetUserOrEmpty(r *http.Request) store.User {
 // SetUserInfo sets user into request context
 func SetUserInfo(r *http.Request, user store.User) *http.Request {
 	u := token.User{
-		ID:      user.ID,
+		ID:       user.ID,
-		Name:    user.Name,
+		Name:     user.Name,
-		Picture: user.Picture,
+		Picture:  user.Picture,
-		IP:      user.IP,
+		IP:       user.IP,
+		Audience: user.SiteID,
 		Attributes: map[string]interface{}{
 			"blocked":  user.Blocked,
 			"verified": user.Verified,

backend/app/rest/user_test.go

@@ -10,14 +10,14 @@ import (
 
 func TestUser_GetUserInfo(t *testing.T) {
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
+	assert.Error(t, err, "no user info")
 
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
+	r = SetUserInfo(r, store.User{Name: "test", ID: "id", SiteID: "test"})
 	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
+	assert.Equal(t, store.User{Name: "test", ID: "id", SiteID: "test"}, u)
 }
 
 func TestUSer_MustGetUserInfo(t *testing.T) {
@@ -28,12 +28,12 @@ func TestUSer_MustGetUserInfo(t *testing.T) {
 	}()
 
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_ = MustGetUserInfo(r)
 	assert.Fail(t, "should panic")
 
 	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
 	u := MustGetUserInfo(r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
 }

backend/app/rpc/client_test.go

@@ -1,97 +0,0 @@
-package rpc
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestClient_Call(t *testing.T) {
-	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
-	defer ts.Close()
-	c := Client{API: ts.URL, Client: http.Client{}}
-	resp, err := c.Call("test", 123, "abc")
-	assert.NoError(t, err)
-	res := ""
-	err = json.Unmarshal(*resp.Result, &res)
-	assert.NoError(t, err)
-	assert.Equal(t, "12345", res)
-	t.Logf("%v %T", res, res)
-}
-
-func TestClient_CallWithObject(t *testing.T) {
-	ts := testServer(t, `{"method":"test","params":{"F1":123,"F2":"abc","F3":"2019-06-09T23:03:55Z"},"id":1}`, `{"result":"12345"}`)
-	defer ts.Close()
-	c := Client{API: ts.URL, Client: http.Client{}}
-	obj := struct {
-		F1 int
-		F2 string
-		F3 time.Time
-	}{
-		F1: 123,
-		F2: "abc",
-		F3: time.Date(2019, 6, 9, 23, 3, 55, 0, time.UTC),
-	}
-
-	resp, err := c.Call("test", obj)
-	assert.NoError(t, err)
-	res := ""
-	err = json.Unmarshal(*resp.Result, &res)
-	assert.NoError(t, err)
-	assert.Equal(t, "12345", res)
-	t.Logf("%v %T", res, res)
-}
-
-func TestClient_CallWithNoParams(t *testing.T) {
-	ts := testServer(t, `{"method":"test","id":1}`, `{"result":"12345"}`)
-	defer ts.Close()
-	c := Client{API: ts.URL, Client: http.Client{}}
-	resp, err := c.Call("test")
-	assert.NoError(t, err)
-	res := ""
-	err = json.Unmarshal(*resp.Result, &res)
-	assert.NoError(t, err)
-	assert.Equal(t, "12345", res)
-	t.Logf("%v %T", res, res)
-}
-
-func TestClient_CallError(t *testing.T) {
-	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`)
-	defer ts.Close()
-	c := Client{API: ts.URL, Client: http.Client{}}
-	_, err := c.Call("test", 123, "abc")
-	assert.EqualError(t, err, "some error")
-}
-
-func TestClient_CallBadResponse(t *testing.T) {
-	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345 invalid}`)
-	defer ts.Close()
-	c := Client{API: ts.URL, Client: http.Client{}}
-	_, err := c.Call("test", 123, "abc")
-	assert.NotNil(t, err)
-}
-
-func TestClient_CallBadRemote(t *testing.T) {
-	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
-	defer ts.Close()
-	c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}
-	_, err := c.Call("test", 123)
-	assert.NotNil(t, err)
-}
-
-func testServer(t *testing.T, req, resp string) *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
-		require.NoError(t, err)
-		assert.Equal(t, req, string(body))
-		t.Logf("req: %s", string(body))
-		fmt.Fprintf(w, resp)
-	}))
-}

backend/app/rpc/remote.go

@@ -1,23 +0,0 @@
-// Package rpc implements client ans server for RPC-like communication with remote storage.
-// The protocol is somewhat simplified version of json-rpc with a single POST call sending
-// Request json (method name and the list of parameters) and receiving back json Response with "result" json
-// and error string
-package rpc
-
-import (
-	"encoding/json"
-)
-
-// Request encloses method name and all params
-type Request struct {
-	Method string      `json:"method"`
-	Params interface{} `json:"params,omitempty"`
-	ID     uint64      `json:"id"`
-}
-
-// Response encloses result and error received from remote server
-type Response struct {
-	Result *json.RawMessage `json:"result,omitempty"`
-	Error  string           `json:"error,omitempty"`
-	ID     uint64           `json:"id"`
-}

backend/app/rpc/server_test.go

@@ -1,245 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestServerPrimitiveTypes(t *testing.T) {
-	s := Server{API: "/v1/cmd"}
-
-	type respData struct {
-		Res1 string
-		Res2 bool
-	}
-
-	s.Add("test", func(id uint64, params json.RawMessage) Response {
-		args := []interface{}{}
-		if err := json.Unmarshal(params, &args); err != nil {
-			return Response{Error: err.Error()}
-		}
-		t.Logf("%+v", args)
-
-		assert.Equal(t, 3, len(args))
-		assert.Equal(t, "blah", args[0].(string))
-		assert.Equal(t, 42., args[1].(float64))
-		assert.Equal(t, true, args[2].(bool))
-
-		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
-		assert.NoError(t, err)
-		return r
-	})
-
-	go func() { _ = s.Run(9091) }()
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-	time.Sleep(10 * time.Millisecond)
-
-	// check with direct http call
-	clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true}, ID: 123}
-	b := bytes.Buffer{}
-	require.NoError(t, json.NewEncoder(&b).Encode(clientReq))
-	resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	data, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-	assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true},"id":123}`+"\n", string(data))
-
-	// check with client call
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
-	r, err := c.Call("test", "blah", 42, true)
-	assert.NoError(t, err)
-	assert.Equal(t, "", r.Error)
-
-	res := respData{}
-	err = json.Unmarshal(*r.Result, &res)
-	assert.NoError(t, err)
-	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
-	assert.Equal(t, uint64(1), r.ID)
-}
-
-func TestServerWithObject(t *testing.T) {
-	s := Server{API: "/v1/cmd"}
-
-	type respData struct {
-		Res1 string
-		Res2 bool
-	}
-
-	type reqData struct {
-		Time time.Time
-		F1   string
-		F2   time.Duration
-	}
-
-	s.Add("test", func(id uint64, params json.RawMessage) Response {
-		arg := reqData{}
-		if err := json.Unmarshal(params, &arg); err != nil {
-			return Response{Error: err.Error()}
-		}
-		t.Logf("%+v", arg)
-
-		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
-		assert.NoError(t, err)
-		return r
-	})
-
-	go func() { _ = s.Run(9091) }()
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-	time.Sleep(10 * time.Millisecond)
-
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
-	r, err := c.Call("test", reqData{Time: time.Now(), F1: "sawert", F2: time.Minute})
-	assert.NoError(t, err)
-	assert.Equal(t, "", r.Error)
-
-	res := respData{}
-	err = json.Unmarshal(*r.Result, &res)
-	assert.NoError(t, err)
-	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
-}
-
-func TestServerMethodNotImplemented(t *testing.T) {
-	s := Server{}
-	ts := httptest.NewServer(http.HandlerFunc(s.handler))
-	defer ts.Close()
-	s.Add("test", func(id uint64, params json.RawMessage) Response {
-		return Response{}
-	})
-
-	r := Request{Method: "blah"}
-	buf := bytes.Buffer{}
-	assert.NoError(t, json.NewEncoder(&buf).Encode(r))
-	resp, err := http.Post(ts.URL, "application/json", &buf)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusNotImplemented, resp.StatusCode)
-
-	assert.EqualError(t, s.Shutdown(), "http server is not running")
-}
-
-func TestServerWithAuth(t *testing.T) {
-	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
-
-	s.Add("test", func(id uint64, params json.RawMessage) Response {
-		args := []interface{}{}
-		if err := json.Unmarshal(params, &args); err != nil {
-			return Response{Error: err.Error()}
-		}
-		t.Logf("%+v", args)
-
-		assert.Equal(t, 3, len(args))
-		assert.Equal(t, "blah", args[0].(string))
-		assert.Equal(t, 42., args[1].(float64))
-		assert.Equal(t, true, args[2].(bool))
-
-		r, err := s.EncodeResponse(id, "res blah", nil)
-		assert.NoError(t, err)
-		return r
-	})
-
-	go func() { _ = s.Run(9091) }()
-	time.Sleep(10 * time.Millisecond)
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
-	r, err := c.Call("test", "blah", 42, true)
-	assert.NoError(t, err)
-	assert.Equal(t, "", r.Error)
-	val := ""
-	err = json.Unmarshal(*r.Result, &val)
-	assert.NoError(t, err)
-	assert.Equal(t, "res blah", val)
-
-	c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
-	_, err = c.Call("test", "blah", 42, true)
-	assert.EqualError(t, err, "bad status 401 Unauthorized for test")
-}
-
-func TestServerErrReturn(t *testing.T) {
-	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
-
-	s.Add("test", func(id uint64, params json.RawMessage) Response {
-		args := []interface{}{}
-		if err := json.Unmarshal(params, &args); err != nil {
-			return Response{Error: err.Error()}
-		}
-		t.Logf("%+v", args)
-
-		assert.Equal(t, 3, len(args))
-		assert.Equal(t, "blah", args[0].(string))
-		assert.Equal(t, 42., args[1].(float64))
-		assert.Equal(t, true, args[2].(bool))
-
-		r, err := s.EncodeResponse(id, "res blah", errors.New("some error"))
-		assert.NoError(t, err)
-		return r
-	})
-
-	go func() { _ = s.Run(9091) }()
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-	time.Sleep(10 * time.Millisecond)
-
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
-	_, err := c.Call("test", "blah", 42, true)
-	assert.EqualError(t, err, "some error")
-}
-
-func TestServerGroup(t *testing.T) {
-	s := Server{API: "/v1/cmd"}
-	s.Group("pre", HandlersGroup{
-		"fn1": func(id uint64, params json.RawMessage) Response {
-			return Response{}
-		},
-		"fn2": func(id uint64, params json.RawMessage) Response {
-			return Response{}
-		},
-	})
-	go func() { _ = s.Run(9091) }()
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-	time.Sleep(10 * time.Millisecond)
-
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
-	_, err := c.Call("fn1")
-	assert.EqualError(t, err, "bad status 501 Not Implemented for fn1")
-
-	_, err = c.Call("pre.fn1")
-	assert.NoError(t, err)
-	_, err = c.Call("pre.fn2")
-	assert.NoError(t, err)
-}
-
-func TestServerAddLate(t *testing.T) {
-	s := Server{API: "/v1/cmd"}
-	s.Add("fn1", func(id uint64, params json.RawMessage) Response {
-		return Response{}
-	})
-	go func() { _ = s.Run(9091) }()
-	defer func() { assert.NoError(t, s.Shutdown()) }()
-	time.Sleep(10 * time.Millisecond)
-
-	// too late, ignored after run
-	s.Add("fn2", func(id uint64, params json.RawMessage) Response {
-		return Response{}
-	})
-
-	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
-	_, err := c.Call("fn1")
-	assert.NoError(t, err)
-	_, err = c.Call("fn2")
-	assert.EqualError(t, err, "bad status 501 Not Implemented for fn2")
-}
-
-func TestServerNoHandlers(t *testing.T) {
-	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
-	assert.EqualError(t, s.Run(9091), "nothing mapped for dispatch, Add has to be called prior to Run")
-}

backend/app/store/admin/admin.go

@@ -3,6 +3,7 @@ package admin
 
 import (
 	"errors"
+	"strings"
 
 	log "github.com/go-pkgz/lgr"
 )
@@ -12,19 +13,33 @@ type Store interface {
 	Key() (key string, err error)
 	Admins(siteID string) (ids []string, err error)
 	Email(siteID string) (email string, err error)
+	Enabled(siteID string) (ok bool, err error)
+	OnEvent(siteID string, et EventType) error
 }
 
+// EventType indicates type of the event
+type EventType int
+
+// enum of all event types
+const (
+	EvCreate EventType = iota
+	EvDelete
+	EvUpdate
+	EvVote
+)
+
 // StaticStore implements keys.Store with a single set of admins and email for all sites
 type StaticStore struct {
 	admins []string
 	email  string
 	key    string
+	sites  []string
 }
 
 // NewStaticStore makes StaticStore instance with given key
-func NewStaticStore(key string, admins []string, email string) *StaticStore {
+func NewStaticStore(key string, sites []string, admins []string, email string) *StaticStore {
 	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
-	return &StaticStore{key: key, admins: admins, email: email}
+	return &StaticStore{key: key, sites: sites, admins: admins, email: email}
 }
 
 // NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
@@ -49,3 +64,19 @@ func (s *StaticStore) Admins(string) (ids []string, err error) {
 func (s *StaticStore) Email(string) (email string, err error) {
 	return s.email, nil
 }
+
+// Enabled if always true for StaticStore
+func (s *StaticStore) Enabled(site string) (ok bool, err error) {
+	if len(s.sites) == 0 {
+		return true, nil
+	}
+	for _, allowedSite := range s.sites {
+		if strings.EqualFold(allowedSite, site) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// OnEvent doesn nothing for StaticStore
+func (s *StaticStore) OnEvent(siteID string, et EventType) error { return nil }

backend/app/store/admin/admin_test.go

@@ -7,17 +7,25 @@ import (
 )
 
 func TestStaticStore_Get(t *testing.T) {
-	var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com")
+	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"}, []string{"123", "xyz"}, "aa@example.com")
 
 	k, err := ks.Key()
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 
-	a, err := ks.Admins("any")
+	a, err := ks.Admins("s1")
 	assert.NoError(t, err)
 	assert.Equal(t, []string{"123", "xyz"}, a)
 
-	email, err := ks.Email("blah")
+	email, err := ks.Email("s2")
 	assert.NoError(t, err)
 	assert.Equal(t, "aa@example.com", email)
+
+	enabled, err := ks.Enabled("s3")
+	assert.NoError(t, err)
+	assert.Equal(t, true, enabled)
+
+	enabled, err = ks.Enabled("serr")
+	assert.NoError(t, err)
+	assert.Equal(t, false, enabled)
 }

backend/app/store/admin/remote.go

@@ -9,12 +9,12 @@ package admin
 import (
 	"encoding/json"
 
-	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/go-pkgz/jrpc"
 )
 
 // RPC implements remote engine and delegates all Calls to remote http server
 type RPC struct {
-	rpc.Client
+	jrpc.Client
 }
 
 // Key returns the key, same for all sites
@@ -53,3 +53,25 @@ func (r *RPC) Email(siteID string) (email string, err error) {
 	}
 	return email, nil
 }
+
+// Enabled returns true if allowed
+func (r *RPC) Enabled(siteID string) (ok bool, err error) {
+	resp, err := r.Call("admin.enabled", siteID)
+	if err != nil {
+		return false, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ok); err != nil {
+		return false, err
+	}
+	return ok, nil
+}
+
+// OnEvent reacts (register) events about data modification
+func (r *RPC) OnEvent(siteID string, et EventType) error {
+	_, err := r.Call("admin.event", siteID, et)
+	if err != nil {
+		return err
+	}
+	return nil
+}

backend/app/store/admin/remote_test.go

@@ -13,17 +13,16 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/go-pkgz/jrpc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/rpc"
 )
 
 func TestRemote_Key(t *testing.T) {
 	ts := testServer(t, `{"method":"admin.key","id":1}`,
 		`{"result":"12345","id":1}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	var a Store = &c
 	_ = a
@@ -35,10 +34,10 @@ func TestRemote_Key(t *testing.T) {
 }
 
 func TestRemote_Admins(t *testing.T) {
-	ts := testServer(t, `{"method":"admin.admins","params":["site-1"],"id":1}`,
+	ts := testServer(t, `{"method":"admin.admins","params":"site-1","id":1}`,
 		`{"result":["id1","id2"],"id":1}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	var a Store = &c
 	_ = a
@@ -50,10 +49,10 @@ func TestRemote_Admins(t *testing.T) {
 }
 
 func TestRemote_Email(t *testing.T) {
-	ts := testServer(t, `{"method":"admin.email","params":["site-1"],"id":1}`,
+	ts := testServer(t, `{"method":"admin.email","params":"site-1","id":1}`,
 		`{"result":"bbb@example.com","id":1}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	var a Store = &c
 	_ = a
@@ -63,6 +62,34 @@ func TestRemote_Email(t *testing.T) {
 	assert.Equal(t, "bbb@example.com", res)
 	t.Logf("%v %T", res, res)
 }
+
+func TestRemote_Enables(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.enabled","params":"site-1","id":1}`,
+		`{"result":true,"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Enabled("site-1")
+	assert.NoError(t, err)
+	assert.True(t, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_OnEvent(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.event","params":["site-1",2],"id":1}`, `{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.OnEvent("site-1", EvUpdate)
+	assert.NoError(t, err)
+}
+
 func testServer(t *testing.T, req, resp string) *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		body, err := ioutil.ReadAll(r.Body)

backend/app/store/comment.go

@@ -11,21 +11,22 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID          string          `json:"id" bson:"_id"`
+	ID          string                 `json:"id" bson:"_id"`
-	ParentID    string          `json:"pid"`
+	ParentID    string                 `json:"pid"`
-	Text        string          `json:"text"`
+	Text        string                 `json:"text"`
-	Orig        string          `json:"orig,omitempty"`
+	Orig        string                 `json:"orig,omitempty"`
-	User        User            `json:"user"`
+	User        User                   `json:"user"`
-	Locator     Locator         `json:"locator"`
+	Locator     Locator                `json:"locator"`
-	Score       int             `json:"score"`
+	Score       int                    `json:"score"`
-	Votes       map[string]bool `json:"votes,omitempty"`
+	Votes       map[string]bool        `json:"votes,omitempty"`
-	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
+	VotedIPs    map[string]VotedIPInfo `json:"voted_ips,omitempty"` // voted ips (hashes) with TS
-	Controversy float64         `json:"controversy,omitempty"`
+	Vote        int                    `json:"vote"`                // vote for the current user, -1/1/0.
-	Timestamp   time.Time       `json:"time" bson:"time"`
+	Controversy float64                `json:"controversy,omitempty"`
-	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Timestamp   time.Time              `json:"time" bson:"time"`
-	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Edit        *Edit                  `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
+	Pin         bool                   `json:"pin,omitempty" bson:"pin,omitempty"`
-	PostTitle   string          `json:"title,omitempty" bson:"title"`
+	Deleted     bool                   `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string                 `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post
@@ -56,6 +57,12 @@ type BlockedUser struct {
 	Until time.Time `json:"time"`
 }
 
+// VotedIPInfo keeps timestamp and voting value (direction). Used as VotedIPs value
+type VotedIPInfo struct {
+	Timestamp time.Time
+	Value     bool
+}
+
 // DeleteMode defines how much comment info will be erased
 type DeleteMode int
 
@@ -106,7 +113,7 @@ func (c *Comment) Sanitize() {
 	c.Text = p.Sanitize(c.Text)
 	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = template.HTMLEscapeString(c.User.Name)
+	c.User.Name = c.escapeHtmlWithSome(c.User.Name)
 	c.User.Picture = p.Sanitize(c.User.Picture)
 }
 
@@ -130,3 +137,11 @@ func (c *Comment) Snippet(limit int) string {
 	}
 	return string(snippet) + " ..."
 }
+
+func (c *Comment) escapeHtmlWithSome(inp string) string {
+	res := template.HTMLEscapeString(inp)
+	res = strings.Replace(res, """, "\"", -1)
+	res = strings.Replace(res, "'", "'", -1)
+	res = strings.Replace(res, "&", "&", -1)
+	return res
+}

backend/app/store/comment_test.go

@@ -28,11 +28,11 @@ func TestComment_Sanitize(t *testing.T) {
 		{
 			inp: Comment{
 				Text: "blah 123" + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 			out: Comment{
 				Text: `blah 123` + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 		},
 		{
@@ -43,6 +43,10 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah & & 123 — —"},
 			out: Comment{Text: `blah & & 123 — —`},
 		},
+		{
+			inp: Comment{Text: "blah & & 123", User: User{Name: "name <> & ' ` \""}},
+			out: Comment{Text: `blah &amp; &amp; 123`, User: User{Name: "name &lt;&gt; & ' ` \""}},
+		},
 	}
 
 	for n, tt := range tbl {

backend/app/store/engine/bolt.go

@@ -16,12 +16,13 @@ import (
 )
 
 // BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
+// there are 6 types of top-level buckets:
 //  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
 //  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
 //    value is not full comment but a reference combined from post-url+commentID
 //  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
 //    is a nested bucket named userID with kv as ts:reference
+//  - users details in "user_details" bucket. Key is userID, value - UserDetailEntry
 //  - blocking info sits in "block" bucket. Key is userID, value - ts
 //  - counts per post to keep number of comments. Key is post url, value - count
 //  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
@@ -31,13 +32,14 @@ type BoltDB struct {
 
 const (
 	// top level buckets
-	postsBucketName    = "posts"
+	postsBucketName       = "posts"
-	lastBucketName     = "last"
+	lastBucketName        = "last"
-	userBucketName     = "users"
+	userBucketName        = "users"
-	blocksBucketName   = "block"
+	userDetailsBucketName = "user_details"
-	infoBucketName     = "info"
+	blocksBucketName      = "block"
-	readonlyBucketName = "readonly"
+	infoBucketName        = "info"
-	verifiedBucketName = "verified"
+	readonlyBucketName    = "readonly"
+	verifiedBucketName    = "verified"
 
 	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
 )
@@ -59,8 +61,8 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
+		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, userDetailsBucketName,
-			readonlyBucketName, verifiedBucketName}
+			blocksBucketName, infoBucketName, readonlyBucketName, verifiedBucketName}
 		err = db.Update(func(tx *bolt.Tx) error {
 			for _, bktName := range topBuckets {
 				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
@@ -82,7 +84,6 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 
 // Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
 func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
 	bdb, err := b.db(comment.Locator.SiteID)
 	if err != nil {
 		return "", err
@@ -205,6 +206,33 @@ func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
 	return b.setFlag(req)
 }
 
+// UserDetail sets or gets single detail value, or gets all details for requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend interface by two separate functions).
+func (b *BoltDB) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
+	switch req.Detail {
+	case UserEmail:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		if req.Update == "" { // read detail value, no update requested
+			return b.getUserDetail(req)
+		}
+
+		return b.setUserDetail(req)
+	case AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			return b.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
 // Update for locator.URL with mutable part of comment
 func (b *BoltDB) Update(comment store.Comment) error {
 
@@ -370,7 +398,7 @@ func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) {
 	return nil, errors.Errorf("flag %s not listable", req.Flag)
 }
 
-// Delete post(s) by id or by userID
+// Delete post(s), user, comment, user details, or everything
 func (b *BoltDB) Delete(req DeleteRequest) error {
 
 	bdb, e := b.db(req.Locator.SiteID)
@@ -379,11 +407,13 @@ func (b *BoltDB) Delete(req DeleteRequest) error {
 	}
 
 	switch {
-	case req.Locator.URL != "" && req.CommentID != "":
+	case req.UserDetail != "": // delete user detail
+		return b.deleteUserDetail(bdb, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
 		return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
-	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
 		return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode)
-	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
 		return b.deleteAll(bdb, req.Locator.SiteID)
 	}
 
@@ -618,6 +648,142 @@ func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error
 	return bkt, nil
 }
 
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (b *BoltDB) getUserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return result, e
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		var entry UserDetailEntry
+		bucket := tx.Bucket([]byte(userDetailsBucketName))
+		value := bucket.Get([]byte(req.UserID))
+		// return no error in case of absent entry
+		if value != nil {
+			if err := json.Unmarshal(value, &entry); err != nil {
+				return errors.Wrap(e, "failed to unmarshal entry")
+			}
+			switch req.Detail {
+			case UserEmail:
+				result = []UserDetailEntry{{UserID: req.UserID, Email: entry.Email}}
+			}
+		}
+		return nil
+	})
+
+	return result, err
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (b *BoltDB) setUserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return result, e
+	}
+
+	var entry UserDetailEntry
+	err = bdb.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(userDetailsBucketName))
+		value := bucket.Get([]byte(req.UserID))
+		// return no error in case of absent entry
+		if value != nil {
+			if err := json.Unmarshal(value, &entry); err != nil {
+				return errors.Wrap(e, "failed to unmarshal entry")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return result, err
+	}
+
+	if entry.UserID == "" {
+		// new entry to be created, need to set UserID for it
+		entry.UserID = req.UserID
+	}
+
+	switch req.Detail {
+	case UserEmail:
+		entry.Email = req.Update
+	}
+
+	err = bdb.Update(func(tx *bolt.Tx) error {
+		err := b.save(tx.Bucket([]byte(userDetailsBucketName)), req.UserID, entry)
+		return errors.Wrapf(err, "failed to update detail %s for %s in %s", req.Detail, req.UserID, req.Locator.SiteID)
+	})
+
+	return []UserDetailEntry{entry}, err
+}
+
+// listDetails lists all available users details for given site
+func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err error) {
+	bdb, e := b.db(loc.SiteID)
+	if e != nil {
+		return result, e
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		var entry UserDetailEntry
+		bucket := tx.Bucket([]byte(userDetailsBucketName))
+		return bucket.ForEach(func(userID, value []byte) error {
+			if err := json.Unmarshal(value, &entry); err != nil {
+				return errors.Wrap(e, "failed to unmarshal entry")
+			}
+			result = append(result, entry)
+			return nil
+		})
+	})
+	return result, err
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry
+func (b *BoltDB) deleteUserDetail(bdb *bolt.DB, userID string, userDetail UserDetail) error {
+	var entry UserDetailEntry
+	err := bdb.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(userDetailsBucketName))
+		value := bucket.Get([]byte(userID))
+		// return no error in case of absent entry
+		if value != nil {
+			if err := json.Unmarshal(value, &entry); err != nil {
+				return errors.Wrap(err, "failed to unmarshal entry")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if entry == (UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case UserEmail:
+		entry.Email = ""
+	case AllUserDetails:
+		entry = UserDetailEntry{UserID: userID}
+	}
+
+	if entry == (UserDetailEntry{UserID: userID}) {
+		// if entry doesn't have non-empty details, we should delete it
+		return bdb.Update(func(tx *bolt.Tx) error {
+			err := tx.Bucket([]byte(userDetailsBucketName)).Delete([]byte(userID))
+			return errors.Wrapf(err, "failed to delete user detail %s for %s", userDetail, userID)
+		})
+	}
+
+	return bdb.Update(func(tx *bolt.Tx) error {
+		// updated entry is not empty and we need to store it's updated copy
+		err := b.save(tx.Bucket([]byte(userDetailsBucketName)), userID, entry)
+		return errors.Wrapf(err, "failed to update detail %s for %s", userDetail, userID)
+	})
+}
+
 func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
 
 	return bdb.Update(func(tx *bolt.Tx) error {
@@ -657,7 +823,7 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
 func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
 
 	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
+	toDelete := []string{postsBucketName, lastBucketName, userBucketName, userDetailsBucketName, infoBucketName}
 
 	// delete top-level buckets
 	err := bdb.Update(func(tx *bolt.Tx) error {
@@ -676,7 +842,7 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
 	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
 }
 
-// deleteUser removes all comments for given user. Everything will be market as deleted
+// deleteUser removes all comments and details for given user. Everything will be market as deleted
 // and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
 func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
 
@@ -723,7 +889,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
 		}
 	}
 
-	//  delete  user bucket in hard mode
+	// delete user bucket in hard mode
 	if mode == store.HardDelete {
 		err = bdb.Update(func(tx *bolt.Tx) error {
 			usersBkt := tx.Bucket([]byte(userBucketName))
@@ -744,7 +910,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode sto
 		return errors.Errorf("unknown user %s", userID)
 	}
 
-	return err
+	return b.deleteUserDetail(bdb, userID, AllUserDetails)
 }
 
 // getPostBucket return bucket with all comments for postURL

backend/app/store/engine/bolt_test.go

@@ -25,13 +25,13 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
 	assert.Equal(t, "user1", res[0].User.ID)
 	t.Log(res[0].ID)
 
 	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	assert.Equal(t, "key id-1 already in store", err.Error())
 
 	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
@@ -59,7 +59,7 @@ func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
 	assert.Equal(t, true, v)
 
 	_, err = b.Create(comment)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
 
 	flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
@@ -85,7 +85,7 @@ func TestBoltDB_Get(t *testing.T) {
 	assert.Equal(t, "some text2", comment.Text)
 
 	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 
 	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
 	assert.EqualError(t, err, `site "bad" not found`)
@@ -98,7 +98,7 @@ func TestBoltDB_Update(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res), "2 records initially")
+	require.Equal(t, 2, len(res), "2 records initially")
 
 	comment := res[0]
 	comment.Text = "abc 123"
@@ -129,13 +129,13 @@ func TestBoltDB_FindLast(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Limit = 1
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(res))
+	require.Equal(t, 1, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Locator.SiteID = "bad"
@@ -151,13 +151,13 @@ func TestBoltDB_FindLastSince(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(res))
+	require.Equal(t, 1, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
@@ -174,13 +174,13 @@ func TestBoltDB_FindInPostSince(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "-time", Since: ts}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(res))
+	require.Equal(t, 1, len(res))
 	assert.Equal(t, "some text2", res[0].Text)
 
 	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
@@ -196,19 +196,19 @@ func TestBoltDB_FindForUser(t *testing.T) {
 	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
 	res, err := b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
 
 	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
+	require.Equal(t, 1, len(res), "allow 1 comment")
 	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
 
 	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
+	require.Equal(t, 1, len(res), "allow 1 comment")
 	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
 
 	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
@@ -223,7 +223,7 @@ func TestBoltDB_FindForUser(t *testing.T) {
 func TestBoltDB_FindForUserPagination(t *testing.T) {
 	_ = os.Remove(testDb)
 	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	defer func() {
 		require.NoError(t, b.Close())
@@ -241,7 +241,7 @@ func TestBoltDB_FindForUserPagination(t *testing.T) {
 		c.Text = fmt.Sprintf("text #%d", i)
 		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
 		_, err = b.Create(c)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}
 
 	// get all comments
@@ -255,7 +255,7 @@ func TestBoltDB_FindForUserPagination(t *testing.T) {
 	req.Limit = 5
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 5, len(res))
+	require.Equal(t, 5, len(res))
 	assert.Equal(t, "id-199", res[0].ID)
 	assert.Equal(t, "id-195", res[4].ID)
 
@@ -263,7 +263,7 @@ func TestBoltDB_FindForUserPagination(t *testing.T) {
 	req.Skip, req.Limit = 10, 3
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 3, len(res))
+	require.Equal(t, 3, len(res))
 	assert.Equal(t, "id-189", res[0].ID)
 	assert.Equal(t, "id-187", res[2].ID)
 
@@ -271,7 +271,7 @@ func TestBoltDB_FindForUserPagination(t *testing.T) {
 	req.Skip, req.Limit = 195, 10
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Equal(t, 5, len(res))
+	require.Equal(t, 5, len(res))
 	assert.Equal(t, "id-4", res[0].ID)
 	assert.Equal(t, "id-0", res[4].ID)
 
@@ -279,7 +279,7 @@ func TestBoltDB_FindForUserPagination(t *testing.T) {
 	req.Skip, req.Limit = 255, 10
 	res, err = b.Find(req)
 	assert.NoError(t, err)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(res))
 }
 
@@ -355,11 +355,11 @@ func TestBoltDB_InfoPost(t *testing.T) {
 
 	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
 	_, err = b.Info(req)
-	require.NotNil(t, err)
+	require.Error(t, err)
 
 	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
 	_, err = b.Info(req)
-	require.NotNil(t, err)
+	require.Error(t, err)
 
 	fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
 	_, err = b.Flag(fr)
@@ -384,7 +384,7 @@ func TestBoltDB_InfoList(t *testing.T) {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err := b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
 
@@ -407,7 +407,7 @@ func TestBoltDB_InfoList(t *testing.T) {
 
 	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
 	res, err = b.Info(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
 
 	req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
@@ -595,24 +595,24 @@ func TestBolt_FlagListBlocked(t *testing.T) {
 		return res
 	}
 	assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
-	assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 150*time.Millisecond))
 	assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
 
 	vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	assert.Equal(t, 2, len(blockedList))
+	require.Equal(t, 2, len(blockedList))
 	assert.Equal(t, "user1", blockedList[0].ID)
 	assert.Equal(t, "user2", blockedList[1].ID)
 	t.Logf("%+v", blockedList)
 
 	// check block expiration
-	time.Sleep(50 * time.Millisecond)
+	time.Sleep(150 * time.Millisecond)
 	vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
 	assert.NoError(t, err)
 	blockedList = toBlocked(vv)
-	assert.Equal(t, 1, len(blockedList))
+	require.Equal(t, 1, len(blockedList))
 	assert.Equal(t, "user1", blockedList[0].ID)
 
 	_, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}})
@@ -620,6 +620,53 @@ func TestBolt_FlagListBlocked(t *testing.T) {
 
 }
 
+func TestBoltDB_UserDetail(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	// add two entries to DB before we start
+	result, err := b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u2", Detail: UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      UserDetailRequest
+		error    string
+		expected []UserDetailEntry
+	}{
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail},
+			expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: UserEmail},
+			error: `site "bad" not found`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1xyz", Detail: UserEmail}},
+		{req: UserDetailRequest{Detail: UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: UserDetailRequest{Detail: UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: UserDetailRequest{Update: "not_relevant", Detail: AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, Detail: AllUserDetails},
+			error: `site "bad" not found`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, Detail: AllUserDetails},
+			expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := b.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
 func TestBolt_DeleteComment(t *testing.T) {
 
 	b, teardown := prep(t)
@@ -642,7 +689,7 @@ func TestBolt_DeleteComment(t *testing.T) {
 
 	res, err = b.Find(reqReq)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "", res[0].Text)
 	assert.True(t, res[0].Deleted, "marked deleted")
 	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
@@ -660,7 +707,7 @@ func TestBolt_DeleteComment(t *testing.T) {
 
 	delReq.CommentID = "123456"
 	err = b.Delete(delReq)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 
 	delReq.Locator.SiteID = "bad"
 	delReq.CommentID = res[0].ID
@@ -680,7 +727,7 @@ func TestBolt_DeleteHard(t *testing.T) {
 	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
 	res, err := b.Find(reqReq)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
+	require.Equal(t, 2, len(res), "initially 2 comments")
 
 	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
 		CommentID: res[0].ID, DeleteMode: store.HardDelete}
@@ -689,7 +736,7 @@ func TestBolt_DeleteHard(t *testing.T) {
 
 	res, err = b.Find(reqReq)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(res))
+	require.Equal(t, 2, len(res))
 	assert.Equal(t, "", res[0].Text)
 	assert.True(t, res[0].Deleted, "marked deleted")
 	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
@@ -713,6 +760,48 @@ func TestBolt_DeleteAll(t *testing.T) {
 	assert.EqualError(t, err, `site "bad" not found`)
 }
 
+func TestBolt_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail, Update: "value1"}
+		readUser   = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail}
+		emailSet   = []UserDetailEntry{{UserID: "user1", Email: "value1"}}
+	)
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	var testData = []struct {
+		delReq    DeleteRequest
+		detailReq UserDetailRequest
+		expected  []UserDetailEntry
+		err       string
+	}{
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: readUser, expected: emailSet, err: `site "bad" not found`},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: readUser},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
+			detailReq: readUser},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		if x.err == "" {
+			require.NoError(t, err, "delete request #%d error", i)
+		} else {
+			require.EqualError(t, err, x.err, "delete request #%d error", i)
+		}
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
 func TestBoltAdmin_DeleteUserHard(t *testing.T) {
 
 	b, teardown := prep(t)
@@ -723,7 +812,7 @@ func TestBoltAdmin_DeleteUserHard(t *testing.T) {
 
 	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
 	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
 	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
 
@@ -735,7 +824,7 @@ func TestBoltAdmin_DeleteUserHard(t *testing.T) {
 	assert.EqualError(t, err, "no comments for user user1 in store")
 
 	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments), "nothing left")
 
 	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
@@ -752,7 +841,7 @@ func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
 
 	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
 	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
 	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
 
@@ -762,7 +851,7 @@ func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
 
 	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
 	assert.NoError(t, err, "no comments for user user1 in store")
-	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
 	assert.True(t, comments[0].Deleted)
 	assert.True(t, comments[1].Deleted)
 	assert.Equal(t, "", comments[0].Text)
@@ -794,7 +883,7 @@ func TestBoltDB_ref(t *testing.T) {
 	assert.Equal(t, "12345", id)
 
 	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestBoltDB_NewFailed(t *testing.T) {
@@ -807,7 +896,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	_ = os.Remove(testDb)
 
 	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	b = boltStore
 
 	comment := store.Comment{
@@ -818,7 +907,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		ID:        "id-2",
@@ -828,7 +917,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	teardown = func() {
 		require.NoError(t, b.Close())

backend/app/store/engine/engine.go

@@ -16,16 +16,19 @@ import (
 
 // Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
+	Create(comment store.Comment) (commentID string, err error)  // create new comment, avoid dups by id
-	Update(comment store.Comment) error                         // update comment, mutable parts only
+	Update(comment store.Comment) error                          // update comment, mutable parts only
-	Get(req GetRequest) (store.Comment, error)                  // get comment by id
+	Get(req GetRequest) (store.Comment, error)                   // get comment by id
-	Find(req FindRequest) ([]store.Comment, error)              // find comments for locator or site
+	Find(req FindRequest) ([]store.Comment, error)               // find comments for locator or site
-	Info(req InfoRequest) ([]store.PostInfo, error)             // get post(s) meta info
+	Info(req InfoRequest) ([]store.PostInfo, error)              // get post(s) meta info
-	Count(req FindRequest) (int, error)                         // get count for post or user
+	Count(req FindRequest) (int, error)                          // get count for post or user
-	Delete(req DeleteRequest) error                             // delete post(s) by id or by userID
+	Delete(req DeleteRequest) error                              // Delete post(s), user, comment, user details, or everything
-	Flag(req FlagRequest) (bool, error)                         // set and get flags
+	Flag(req FlagRequest) (bool, error)                          // set and get flags
-	ListFlags(req FlagRequest) ([]interface{}, error)           // get list of flagged keys, like blocked & verified user
+	ListFlags(req FlagRequest) ([]interface{}, error)            // get list of flagged keys, like blocked & verified user
-	Close() error                                               // close storage engine
+	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) // sets or gets single detail value, or gets all details for requested site.
+	// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+	// and all site's details listing under the same function (and not to extend interface by two separate functions).
+	Close() error // close storage engine
 }
 
 // GetRequest is the input for Get func
@@ -57,6 +60,7 @@ type DeleteRequest struct {
 	Locator    store.Locator    `json:"locator"` // lack of URL means site operation
 	CommentID  string           `json:"comment_id,omitempty"`
 	UserID     string           `json:"user_id,omitempty"`
+	UserDetail UserDetail       `json:"user_detail,omitempty"`
 	DeleteMode store.DeleteMode `json:"del_mode"`
 }
 
@@ -79,6 +83,11 @@ const (
 	Verified = Flag("verified")
 	Blocked  = Flag("blocked")
 )
+const (
+	// All possible user details
+	UserEmail      = UserDetail("email")
+	AllUserDetails = UserDetail("all") // used for listing and deletion requests
+)
 
 // FlagRequest is the input for both get/set for flags, like blocked, verified and so on
 type FlagRequest struct {
@@ -89,6 +98,23 @@ type FlagRequest struct {
 	TTL     time.Duration `json:"ttl,omitempty"`     // ttl for time-sensitive flags only, like blocked for some period
 }
 
+// UserDetail defines name of the user detail
+type UserDetail string
+
+// UserDetailEntry contains single user details entry
+type UserDetailEntry struct {
+	UserID string `json:"user_id"`         // duplicate user's id to use this structure not only embedded but separately
+	Email  string `json:"email,omitempty"` // UserEmail
+}
+
+// UserDetailRequest is the input for both get/set for details, like email
+type UserDetailRequest struct {
+	Detail  UserDetail    `json:"detail"`           // detail name
+	Locator store.Locator `json:"locator"`          // post locator
+	UserID  string        `json:"user_id"`          // user id for get\set
+	Update  string        `json:"update,omitempty"` // update value
+}
+
 const (
 	// limits
 	lastLimit = 1000

backend/app/store/engine/engine_mock.go

@@ -1,8 +1,11 @@
 // Code generated by mockery v1.0.0. DO NOT EDIT.
+
 package engine
 
-import mock "github.com/stretchr/testify/mock"
+import (
-import store "github.com/umputun/remark/backend/app/store"
+	mock "github.com/stretchr/testify/mock"
+	store "github.com/umputun/remark/backend/app/store"
+)
 
 // MockInterface is an autogenerated mock type for the Interface type
 type MockInterface struct {
@@ -203,3 +206,26 @@ func (_m *MockInterface) Update(comment store.Comment) error {
 
 	return r0
 }
+
+// UserDetail provides a mock function with given fields: req
+func (_m *MockInterface) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
+	ret := _m.Called(req)
+
+	var r0 []UserDetailEntry
+	if rf, ok := ret.Get(0).(func(UserDetailRequest) []UserDetailEntry); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]UserDetailEntry)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(UserDetailRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

backend/app/store/engine/remote.go

@@ -3,13 +3,14 @@ package engine
 import (
 	"encoding/json"
 
-	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/go-pkgz/jrpc"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
 // RPC implements remote engine and delegates all Calls to remote http server
 type RPC struct {
-	rpc.Client
+	jrpc.Client
 }
 
 // Create comment and return ID
@@ -81,6 +82,18 @@ func (r *RPC) ListFlags(req FlagRequest) (list []interface{}, err error) {
 	return list, err
 }
 
+// UserDetail sets or gets single detail value, or gets all details for requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend interface by two separate functions).
+func (r *RPC) UserDetail(req UserDetailRequest) (result []UserDetailEntry, err error) {
+	resp, err := r.Call("store.user_detail", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &result)
+	return result, err
+}
+
 // Count gets comments count by user or site
 func (r *RPC) Count(req FindRequest) (count int, err error) {
 	resp, err := r.Call("store.count", req)
@@ -91,7 +104,7 @@ func (r *RPC) Count(req FindRequest) (count int, err error) {
 	return count, err
 }
 
-// Delete post(s) by id or by userID
+// Delete post(s), user, comment, user details, or everything
 func (r *RPC) Delete(req DeleteRequest) error {
 	_, err := r.Call("store.delete", req)
 	return err

backend/app/store/engine/remote_test.go

@@ -5,14 +5,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/jrpc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rpc"
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -20,7 +19,7 @@ func TestRemote_Create(t *testing.T) {
 	ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`,
 		`{"result":"12345","id":1}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	var eng Interface = &c
 	_ = eng
@@ -35,7 +34,7 @@ func TestRemote_Create(t *testing.T) {
 func TestRemote_Get(t *testing.T) {
 	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
 	res, err := c.Get(req)
@@ -47,7 +46,7 @@ func TestRemote_Get(t *testing.T) {
 func TestRemote_GetWithErrorResult(t *testing.T) {
 	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
 	_, err := c.Get(req)
@@ -57,7 +56,7 @@ func TestRemote_GetWithErrorResult(t *testing.T) {
 func TestRemote_GetWithErrorDecode(t *testing.T) {
 	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
 	_, err := c.Get(req)
@@ -65,12 +64,12 @@ func TestRemote_GetWithErrorDecode(t *testing.T) {
 }
 
 func TestRemote_GetWithErrorRemote(t *testing.T) {
-	c := RPC{Client: rpc.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}}
+	c := RPC{Client: jrpc.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}}
 
 	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
 	_, err := c.Get(req)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
-	assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error())
+	assert.Contains(t, err.Error(), "remote call failed for store.get:")
 }
 
 func TestRemote_FailedStatus(t *testing.T) {
@@ -81,7 +80,7 @@ func TestRemote_FailedStatus(t *testing.T) {
 		w.WriteHeader(400)
 	}))
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
 	_, err := c.Get(req)
@@ -91,7 +90,7 @@ func TestRemote_FailedStatus(t *testing.T) {
 func TestRemote_Update(t *testing.T) {
 	ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	err := c.Update(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"},
 		Text: "msg"})
@@ -102,7 +101,7 @@ func TestRemote_Update(t *testing.T) {
 func TestRemote_Find(t *testing.T) {
 	ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	res, err := c.Find(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10})
 	assert.NoError(t, err)
@@ -112,7 +111,7 @@ func TestRemote_Find(t *testing.T) {
 func TestRemote_Info(t *testing.T) {
 	ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	res, err := c.Info(InfoRequest{Locator: store.Locator{URL: "http://example.com/url"},
 		Limit: 10, Skip: 5, ReadOnlyAge: 10})
@@ -123,7 +122,7 @@ func TestRemote_Info(t *testing.T) {
 func TestRemote_Flag(t *testing.T) {
 	ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	res, err := c.Flag(FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: Verified})
 	assert.NoError(t, err)
@@ -133,16 +132,38 @@ func TestRemote_Flag(t *testing.T) {
 func TestRemote_ListFlag(t *testing.T) {
 	ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 	res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked})
 	assert.NoError(t, err)
 	assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
 }
 
+func TestRemote_UserDetail(t *testing.T) {
+	ts := testServer(t, `{"method":"store.user_detail","params":{"detail":"email","locator":{"url":"http://example.com/url"},"user_id":"username"},"id":1}`, `{"result":[{"user_id":"u1","email":"test_email@example.com"}]}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := UserDetailRequest{Locator: store.Locator{URL: "http://example.com/url"}, UserID: "username", Detail: UserEmail}
+	res, err := c.UserDetail(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []UserDetailEntry{{UserID: "u1", Email: "test_email@example.com"}}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_UserDetailWithErrorResult(t *testing.T) {
+	ts := testServer(t, `{"method":"store.user_detail","params":{"detail":"email","locator":{"url":"http://example.com/url"},"user_id":"username","update":"new_value@example.com"},"id":1}`, `{"error":"failed"}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := UserDetailRequest{Locator: store.Locator{URL: "http://example.com/url"}, UserID: "username", Detail: UserEmail, Update: "new_value@example.com"}
+	_, err := c.UserDetail(req)
+	assert.EqualError(t, err, "failed")
+}
+
 func TestRemote_Count(t *testing.T) {
 	ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	res, err := c.Count(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}})
 	assert.NoError(t, err)
@@ -153,7 +174,7 @@ func TestRemote_Delete(t *testing.T) {
 	ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`,
 		`{}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 
 	err := c.Delete(DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}})
 	assert.NoError(t, err)
@@ -162,7 +183,7 @@ func TestRemote_Delete(t *testing.T) {
 func TestRemote_Close(t *testing.T) {
 	ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
 	defer ts.Close()
-	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 	err := c.Close()
 	assert.NoError(t, err)
 }
@@ -173,6 +194,6 @@ func testServer(t *testing.T, req, resp string) *httptest.Server {
 		require.NoError(t, err)
 		assert.Equal(t, req, string(body))
 		t.Logf("req: %s", string(body))
-		fmt.Fprintf(w, resp)
+		_, _ = fmt.Fprint(w, resp)
 	}))
 }

backend/app/store/image/bolt_store.go

@@ -0,0 +1,169 @@
+package image
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"io"
+	"io/ioutil"
+	"path"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+const imagesStagedBktName = "imagesStaged"
+const imagesBktName = "images"
+const insertTimeBktName = "insertTimestamps"
+
+// Bolt provides image Store for images keeping data in bolt DB, restricts max size.
+// It uses 3 buckets to manage images data.
+// Two buckets contains image data (staged and committed images). Third bucket holds insertion timestamps.
+type Bolt struct {
+	fileName  string
+	db        *bolt.DB
+	MaxSize   int
+	MaxHeight int
+	MaxWidth  int
+}
+
+// Create Bolt Store.
+func NewBoltStorage(fileName string, maxSize int, maxHeight int, maxWidth int, options bolt.Options) (*Bolt, error) {
+	db, err := bolt.Open(fileName, 0600, &options)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
+	}
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		if _, e := tx.CreateBucketIfNotExists([]byte(imagesBktName)); e != nil {
+			return errors.Wrapf(e, "failed to create top level bucket %s", imagesBktName)
+		}
+		if _, e := tx.CreateBucketIfNotExists([]byte(imagesStagedBktName)); e != nil {
+			return errors.Wrapf(e, "failed to create top level bucket %s", imagesStagedBktName)
+		}
+		if _, e := tx.CreateBucketIfNotExists([]byte(insertTimeBktName)); e != nil {
+			return errors.Wrapf(e, "failed to create top level bucket %s", insertTimeBktName)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
+	}
+	return &Bolt{
+		db:        db,
+		fileName:  fileName,
+		MaxSize:   maxSize,
+		MaxHeight: maxHeight,
+		MaxWidth:  maxWidth,
+	}, nil
+}
+
+// SaveWithID saves data from reader with given id
+func (b *Bolt) SaveWithID(id string, r io.Reader) (string, error) {
+	data, err := readAndValidateImage(r, b.MaxSize)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't load image with ID %s", id)
+	}
+
+	data = resize(data, b.MaxWidth, b.MaxHeight)
+
+	err = b.db.Update(func(tx *bolt.Tx) error {
+		if err = tx.Bucket([]byte(imagesStagedBktName)).Put([]byte(id), data); err != nil {
+			return errors.Wrapf(err, "can't put to bucket with %s", id)
+		}
+		tsBuf := &bytes.Buffer{}
+		if err = binary.Write(tsBuf, binary.LittleEndian, time.Now().UnixNano()); err != nil {
+			return errors.Wrapf(err, "can't serialize timestamp for %s", id)
+		}
+		if err = tx.Bucket([]byte(insertTimeBktName)).Put([]byte(id), tsBuf.Bytes()); err != nil {
+			return errors.Wrapf(err, "can't put to bucket with %s", id)
+		}
+		return err
+	})
+
+	return id, err
+}
+
+// Save data from reader to staging bucket in DB
+func (b *Bolt) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+	id = path.Join(userID, guid())
+	return b.SaveWithID(id, r)
+}
+
+// Commit file stored in staging bucket by copying it to permanent bucket
+// Data from staging bucket not removed immediately, but would be removed on cleanup
+func (b *Bolt) Commit(id string) error {
+	err := b.db.Update(func(tx *bolt.Tx) error {
+		data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
+		if data == nil {
+			return errors.Errorf("failed to commit %s, not found in staging", id)
+		}
+		err := tx.Bucket([]byte(imagesBktName)).Put([]byte(id), data)
+		return errors.Wrapf(err, "can't put to bucket with %s", id)
+	})
+	return err
+}
+
+// Load image from DB
+// returns ReadCloser and caller should call close after processing completed.
+func (b *Bolt) Load(id string) (io.ReadCloser, int64, error) {
+	buf := &bytes.Buffer{}
+	var size int = 0
+	err := b.db.View(func(tx *bolt.Tx) error {
+		data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
+		if data == nil {
+			data = tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
+		}
+		if data == nil {
+			return errors.Errorf("can't load image %s", id)
+		}
+		var err error
+		size, err = buf.Write(data)
+		return errors.Wrapf(err, "failed to write for %s", id)
+	})
+	return ioutil.NopCloser(buf), int64(size), err
+}
+
+// Cleanup runs scan of staging and removes old data based on ttl
+func (b *Bolt) Cleanup(ctx context.Context, ttl time.Duration) error {
+	err := b.db.Update(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(insertTimeBktName)).Cursor()
+
+		idsToRemove := [][]byte{}
+
+		for id, tsData := c.First(); id != nil; id, tsData = c.Next() {
+			var ts int64
+			err := binary.Read(bytes.NewReader(tsData), binary.LittleEndian, &ts)
+			if err != nil {
+				return errors.Wrapf(err, "failed to deserialize timestamp for %s", id)
+			}
+
+			age := time.Since(time.Unix(0, ts))
+
+			if age > ttl {
+				log.Printf("[INFO] remove staging image %s, age %v", id, age)
+				idsToRemove = append(idsToRemove, id)
+				err := c.Delete()
+				if err != nil {
+					return errors.Wrapf(err, "failed to remove timestamp for %s", id)
+				}
+			}
+		}
+		imgBkt := tx.Bucket([]byte(imagesStagedBktName))
+		for _, id := range idsToRemove {
+			err := imgBkt.Delete(id)
+			if err != nil {
+				return errors.Wrapf(err, "failed to remove image for %s", id)
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+// SizeLimit returns max size of allowed image
+func (b *Bolt) SizeLimit() int {
+	return b.MaxSize
+}

backend/app/store/image/bolt_store_test.go

@@ -0,0 +1,151 @@
+package image
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBoltStore_SaveCommit(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1")
+	t.Log(id)
+
+	err = svc.db.View(func(tx *bolt.Tx) error {
+		data := tx.Bucket([]byte(imagesStagedBktName)).Get([]byte(id))
+		assert.NotNil(t, data)
+		assert.Equal(t, 1462, len(data))
+		return nil
+	})
+	assert.NoError(t, err)
+
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	err = svc.db.View(func(tx *bolt.Tx) error {
+		data := tx.Bucket([]byte(imagesBktName)).Get([]byte(id))
+		assert.NotNil(t, data)
+		assert.Equal(t, 1462, len(data))
+		return nil
+	})
+	assert.NoError(t, err)
+
+}
+
+func TestBoltStore_LoadAfterSave(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1")
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+
+	_, _, err = svc.Load("abcd")
+	assert.Error(t, err)
+}
+
+func TestBoltStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	save := func(file string, user string) (id string) {
+		id, err := svc.Save(file, user, gopherPNG())
+		require.NoError(t, err)
+
+		checkBoltImgData(t, svc.db, imagesStagedBktName, id, func(data []byte) error {
+			require.NotNil(t, data)
+			assert.Equal(t, 1462, len(data))
+			return nil
+		})
+		return id
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1")
+	img1ts := time.Now()
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2")
+
+	err := svc.Cleanup(context.Background(), time.Since(img1ts)) // clean first images
+	assert.NoError(t, err)
+
+	assertBoltImgNil(t, svc.db, imagesStagedBktName, img1)
+	assertBoltImgNil(t, svc.db, imagesBktName, img1)
+	assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img2)
+	assertBoltImgNotNil(t, svc.db, imagesStagedBktName, img3)
+
+	err = svc.Commit(img3)
+	require.NoError(t, err)
+
+	err = svc.Cleanup(context.Background(), time.Millisecond*10)
+	assert.NoError(t, err)
+
+	assertBoltImgNil(t, svc.db, imagesStagedBktName, img2)
+	assertBoltImgNil(t, svc.db, imagesBktName, img2)
+	assertBoltImgNotNil(t, svc.db, imagesBktName, img3)
+	assert.NoError(t, err)
+}
+
+func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) {
+	checkBoltImgData(t, db, bucket, id, func(data []byte) error {
+		assert.Nil(t, data, id)
+		return nil
+	})
+}
+
+func assertBoltImgNotNil(t *testing.T, db *bolt.DB, bucket string, id string) {
+	checkBoltImgData(t, db, bucket, id, func(data []byte) error {
+		assert.NotNil(t, data, id)
+		return nil
+	})
+}
+
+func checkBoltImgData(t *testing.T, db *bolt.DB, bucket string, id string, callback func([]byte) error) {
+	err := db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(bucket))
+		assert.NotNil(t, bkt, "bucket %s not found", bucket)
+		data := bkt.Get([]byte(id))
+		return callback(data)
+	})
+	assert.NoError(t, err)
+}
+
+func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	svc, err = NewBoltStorage(path.Join(loc, "picture.db"), 1500, 0, 0, bolt.Options{})
+	assert.NoError(t, err, "new bolt storage")
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/fs_store.go

@@ -36,45 +36,39 @@ type FileSystem struct {
 	}
 }
 
-// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// SaveWithID saves data from reader with given id
-// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) SaveWithID(id string, r io.Reader) (string, error) {
-func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+	data, err := readAndValidateImage(r, f.MaxSize)
-
-	lr := io.LimitReader(r, int64(f.MaxSize)+1)
-	data, err := ioutil.ReadAll(lr)
 	if err != nil {
-		return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
+		return "", errors.Wrapf(err, "can't load image with ID %s", id)
-	}
-	if len(data) > f.MaxSize {
-		return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
 	}
 
-	// read header first, needs it to check if data is valid png/gif/jpeg
+	data = resize(data, f.MaxWidth, f.MaxHeight)
-	if !isValidImage(data[:512]) {
-		return "", errors.Errorf("file %s is not in allowed format", fileName)
-	}
-
-	data, resized := resize(data, f.MaxWidth, f.MaxHeight)
-
-	id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
 	dst := f.location(f.Staging, id)
-	if resized { // resized also converted to png
-		id = strings.TrimSuffix(id, filepath.Ext(id)) + ".png"
-		dst = f.location(f.Staging, id)
-	}
 
 	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
 		return "", errors.Wrap(err, "can't make image directory")
 	}
 
 	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
-		return "", errors.Wrapf(err, "can't write image file %s", dst)
+		return "", errors.Wrapf(err, "can't write file")
 	}
 
-	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, fileName, len(data))
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, id, len(data))
 	return id, nil
 }
 
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+	id = path.Join(userID, guid()) // make id as user/uuid
+	finalID, err := f.SaveWithID(id, r)
+	if err != nil {
+		err = errors.Wrapf(err, "can't save file %s", fileName)
+	}
+	return finalID, err
+}
+
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
 	log.Printf("[DEBUG] commit image %s", id)

backend/app/store/image/fs_store_test.go

@@ -47,7 +47,6 @@ func TestFsStore_Save(t *testing.T) {
 	id, err := svc.Save("file1.png", "user1", gopherPNG())
 	assert.NoError(t, err)
 	assert.Contains(t, id, "user1/")
-	assert.Contains(t, id, ".png")
 	t.Log(id)
 
 	img := svc.location(svc.Staging, id)
@@ -65,7 +64,6 @@ func TestFsStore_SaveWithResize(t *testing.T) {
 	id, err := svc.Save("file1.png", "user1", gopherPNG())
 	assert.NoError(t, err)
 	assert.Contains(t, id, "user1/")
-	assert.Contains(t, id, ".png")
 	t.Log(id)
 
 	img := svc.location(svc.Staging, id)
@@ -87,7 +85,6 @@ func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
 	id, err := svc.Save("circles.jpg", "user1", fh)
 	assert.NoError(t, err)
 	assert.Contains(t, id, "user1/")
-	assert.Contains(t, id, ".png")
 	t.Log(id)
 
 	img := svc.location(svc.Staging, id)
@@ -109,7 +106,6 @@ func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
 	id, err := svc.Save("circles.jpg", "user1", fh)
 	assert.NoError(t, err)
 	assert.Contains(t, id, "user1/")
-	assert.Contains(t, id, ".jpg")
 	t.Log(id)
 
 	img := svc.location(svc.Staging, id)
@@ -124,7 +120,7 @@ func TestFsStore_WrongFormat(t *testing.T) {
 	defer teardown()
 
 	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
-	assert.EqualError(t, err, "file file1.png is not in allowed format")
+	assert.Error(t, err)
 }
 
 func TestFsStore_SaveAndCommit(t *testing.T) {
@@ -138,7 +134,7 @@ func TestFsStore_SaveAndCommit(t *testing.T) {
 
 	imgStaging := svc.location(svc.Staging, id)
 	_, err = os.Stat(imgStaging)
-	assert.NotNil(t, err, "no file on staging anymore")
+	assert.Error(t, err, "no file on staging anymore")
 
 	img := svc.location(svc.Location, id)
 	t.Log(img)
@@ -173,7 +169,7 @@ func TestFsStore_LoadAfterSave(t *testing.T) {
 	assert.Equal(t, 1462, len(data))
 	assert.Equal(t, int64(1462), sz)
 	_, _, err = svc.Load("abcd")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestFsStore_LoadAfterCommit(t *testing.T) {
@@ -195,7 +191,7 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
 	assert.Equal(t, 1462, len(data))
 	assert.Equal(t, int64(1462), sz)
 	_, _, err = svc.Load("abcd")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestFsStore_location(t *testing.T) {
@@ -268,9 +264,15 @@ func TestFsStore_Cleanup(t *testing.T) {
 	assert.NoError(t, err)
 
 	_, err = os.Stat(img1)
-	assert.NotNil(t, err, "no file on staging anymore")
+	assert.Error(t, err, "no file on staging anymore")
+	// sometimes two images for user1 are put into same directory, which means that
+	// after first image cleanup it's not empty and won't be deleted
 	_, err = os.Stat(path.Dir(img1))
-	assert.NotNil(t, err, "no dir %s on staging anymore", path.Dir(img1))
+	if path.Dir(img1) != path.Dir(img2) {
+		assert.Error(t, err, "no dir %s on staging anymore", path.Dir(img1))
+	} else {
+		assert.NoError(t, err, "dir %s still on staging", path.Dir(img1))
+	}
 
 	_, err = os.Stat(img2)
 	assert.NoError(t, err, "file on staging")
@@ -282,9 +284,9 @@ func TestFsStore_Cleanup(t *testing.T) {
 	assert.NoError(t, err)
 
 	_, err = os.Stat(img2)
-	assert.NotNil(t, err, "no file on staging anymore")
+	assert.Error(t, err, "no file on staging anymore")
 	_, err = os.Stat(img3)
-	assert.NotNil(t, err, "no file on staging anymore")
+	assert.Error(t, err, "no file on staging anymore")
 }
 
 func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {

backend/app/store/image/image.go

@@ -13,6 +13,7 @@ import (
 	_ "image/jpeg"
 	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"sync"
@@ -29,6 +30,7 @@ import (
 // Store defines interface for saving and loading pictures.
 // Declares two-stage save with commit
 type Store interface {
+	SaveWithID(id string, r io.Reader) (string, error)
 	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
 	Commit(id string) error                                                  // move image from staging to permanent
 	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
@@ -138,23 +140,23 @@ func (s *Service) Close() {
 // resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
 // biggest side (width or height) preserving aspect ratio.
 // Returns original data if resizing is not needed or failed.
-// If resized the result will be for png format and ok flag will be true.
+// If resized the result will be for png format
-func resize(data []byte, limitW, limitH int) ([]byte, bool) {
+func resize(data []byte, limitW, limitH int) []byte {
 	if data == nil || limitW <= 0 || limitH <= 0 {
-		return data, false
+		return data
 	}
 
 	src, _, err := image.Decode(bytes.NewBuffer(data))
 	if err != nil {
 		log.Printf("[WARN] can't decode image, %s", err)
-		return data, false
+		return data
 	}
 
 	bounds := src.Bounds()
 	w, h := bounds.Dx(), bounds.Dy()
 	if w <= limitW && h <= limitH || w <= 0 || h <= 0 {
 		log.Printf("[DEBUG] resizing image is smaller that the limit or has 0 size")
-		return data, false
+		return data
 	}
 
 	newW, newH := getProportionalSizes(w, h, limitW, limitH)
@@ -164,9 +166,9 @@ func resize(data []byte, limitW, limitH int) ([]byte, bool) {
 	var out bytes.Buffer
 	if err = png.Encode(&out, m); err != nil {
 		log.Printf("[WARN] can't encode resized image to png, %s", err)
-		return data, false
+		return data
 	}
-	return out.Bytes(), true
+	return out.Bytes()
 }
 
 // getProportionalSizes returns width and height resized by both dimensions proportionally
@@ -195,6 +197,25 @@ func isValidImage(b []byte) bool {
 	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
 }
 
+func readAndValidateImage(r io.Reader, maxSize int) ([]byte, error) {
+	lr := io.LimitReader(r, int64(maxSize)+1)
+	data, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(data) > maxSize {
+		return nil, errors.Errorf("file is too large (limit=%d)", maxSize)
+	}
+
+	// read header first, needs it to check if data is valid png/gif/jpeg
+	if !isValidImage(data[:512]) {
+		return nil, errors.Errorf("file format is not allowed")
+	}
+
+	return data, nil
+}
+
 // guid makes a globally unique id
 func guid() string {
 	return xid.New().String()

backend/app/store/image/image_mock.go

@@ -1,10 +1,15 @@
 // Code generated by mockery v1.0.0. DO NOT EDIT.
+
 package image
 
-import context "context"
+import (
-import io "io"
+	context "context"
-import mock "github.com/stretchr/testify/mock"
+	io "io"
-import time "time"
+
+	mock "github.com/stretchr/testify/mock"
+
+	time "time"
+)
 
 // MockStore is an autogenerated mock type for the Store type
 type MockStore struct {
@@ -90,6 +95,27 @@ func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string,
 	return r0, r1
 }
 
+// SaveWithID provides a mock function with given fields: id, r
+func (_m *MockStore) SaveWithID(id string, r io.Reader) (string, error) {
+	ret := _m.Called(id, r)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, io.Reader) string); ok {
+		r0 = rf(id, r)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, io.Reader) error); ok {
+		r1 = rf(id, r)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // SizeLimit provides a mock function with given fields:
 func (_m *MockStore) SizeLimit() int {
 	ret := _m.Called()

backend/app/store/image/image_test.go

@@ -20,7 +20,7 @@ func TestService_ExtractPictures(t *testing.T) {
 <img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
 	ids, err := svc.ExtractPictures(html)
 	require.NoError(t, err)
-	assert.Equal(t, 2, len(ids), "two images")
+	require.Equal(t, 2, len(ids), "two images")
 	assert.Equal(t, "user1/pic1.png", ids[0])
 	assert.Equal(t, "user2/pic3.png", ids[1])
 }
@@ -31,7 +31,7 @@ func TestService_ExtractPictures2(t *testing.T) {
 		"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
 	ids, err := svc.ExtractPictures(html)
 	require.NoError(t, err)
-	assert.Equal(t, 1, len(ids), "one image in")
+	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
 }
 
@@ -85,21 +85,18 @@ func TestService_SubmitDelay(t *testing.T) {
 func TestService_resize(t *testing.T) {
 
 	// Reader is nil.
-	resized, ok := resize(nil, 100, 100)
+	resized := resize(nil, 100, 100)
 	assert.Nil(t, resized)
-	assert.False(t, ok)
 
 	// Negative limit error.
-	resized, ok = resize([]byte("some picture bin data"), -1, -1)
+	resized = resize([]byte("some picture bin data"), -1, -1)
 	require.NotNil(t, resized)
 	assert.Equal(t, resized, []byte("some picture bin data"))
-	assert.False(t, ok)
 
 	// Decode error.
-	resized, ok = resize([]byte("invalid image content"), 100, 100)
+	resized = resize([]byte("invalid image content"), 100, 100)
 	assert.NotNil(t, resized)
 	assert.Equal(t, resized, []byte("invalid image content"))
-	assert.False(t, ok)
 
 	cases := []struct {
 		file   string
@@ -111,21 +108,19 @@ func TestService_resize(t *testing.T) {
 
 	for _, c := range cases {
 		img, err := ioutil.ReadFile(c.file)
-		require.Nil(t, err, "can't open test file %s", c.file)
+		require.NoError(t, err, "can't open test file %s", c.file)
 
 		// No need for resize, image dimensions are smaller than resize limit.
-		resized, ok = resize(img, 800, 800)
+		resized = resize(img, 800, 800)
 		assert.NotNil(t, resized, "file %s", c.file)
 		assert.Equal(t, resized, img)
-		assert.False(t, ok)
 
 		// Resizing to half of width. Check resized image format PNG.
-		resized, ok = resize(img, 400, 400)
+		resized = resize(img, 400, 400)
 		assert.NotNil(t, resized, "file %s", c.file)
-		assert.True(t, ok)
 
 		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
-		assert.Nil(t, err, "file %s", c.file)
+		assert.NoError(t, err, "file %s", c.file)
 		assert.Equal(t, "png", format, "file %s", c.file)
 		bounds := imgRz.Bounds()
 		assert.Equal(t, c.wr, bounds.Dx(), "file %s", c.file)