Add subscription link support to notification email verification template (#573)

* add subscription link support to notification email verification template

* always show token for email subscription in verification email

* hide SubscribeURL from users

.dockerignore

@@ -20,4 +20,5 @@ debug.test
 *.prof
 *.test
 remark42
-/backend/var/
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -1,101 +1,103 @@
-workspace:
-  base: /go
-  path: src/github.com/umputun/remark
+kind: pipeline
+name: default
+type: docker
 
-pipeline:
-  
-  build:
-    image: golang:1.12-alpine
+steps:
+
+  - name: build server
+    image: umputun/baseimage:buildgo-latest
     commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
       - cd backend/app
-      - go build -v ./...
+      - go build -mod=vendor
+      - echo "build completed"
 
-  docker_master:
+  - name: docker master
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
     when:
-      branch: [master, release/*]
+      branch: [master]
       event: push
 
-  docker_tag:
+  - name: docker tag
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-    tags: 
-      - ${DRONE_TAG}
-      - latest
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+      tags:
+        - ${DRONE_TAG}
+        - latest
     when:
       event: tag
 
-  artifacts_tag:
+  - name: docker branch
     image: plugins/docker
-    dockerfile: Dockerfile.artifacts
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
-    when:
-      event: tag
-
-  docker_branch:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
+      dry_run: true
     when:
       branch:
         exclude: [master, release/*]
         event: push
 
-  deploy:
-    image: appleboy/drone-ssh
-    host: remark42.com
-    username: umputun
-    port: 22
-    secrets: [ ssh_key ]
-    script:
-      - cd /srv/remark
-      - docker-compose pull
-      - docker-compose up -d
+  - name: artifacts tag
+    image: plugins/docker
+    settings:
+      dockerfile: Dockerfile.artifacts
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - GITHUB_TOKEN=${GITHUB_TOKEN}
+    when:
+      event: tag
+
+  - name: deploy
+    image: docker.umputun.com/system/deploy-ci:master
+    commands:
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
     when:
       branch: master
       event: push
 
-  notify:
+  - name: notify
     image: drillster/drone-email
-    host: smtp.mailgun.org
-    port: 25
-    username: email_username
-    password: email_password
-    from: drone@mg.umputun.com
-    recipients: [ umputun@gmail.com ]
-    secrets: [ email_username, email_password ]
+    settings:
+      host: smtp.mailgun.org
+      username:
+        from_secret: email_username
+      password:
+        from_secret: email_password
+      from: drone@mg.umputun.dev
+      recipients: [ sys@umputun.dev ]
     when:
       status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*       @umputun

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+#patreon: remark42

.github/workflows/ci-build.yml

@@ -0,0 +1,31 @@
+name: build
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: build docker image
+        run: docker build --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github .

.github/workflows/ci-test-backend.yml

@@ -0,0 +1,67 @@
+name: test_backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+      - '.golangci.yml'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+      - '.golangci.yml'
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: debug if needed
+        run: |
+          export DEBUG=${DEBUG:-false}
+          if [[ "$DEBUG" == "true" ]]; then
+            env
+          fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: test backend
+        run: |
+          export TZ="America/Chicago"
+          date
+          cd backend/app
+          go test -mod=vendor -timeout=60s -covermode=count -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          cd ../_example/memory_store
+          go test -race ./...
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.20.0
+          go get -u github.com/mattn/goveralls
+
+      - name: run backend linters
+        run: $GITHUB_WORKSPACE/golangci-lint run --config .golangci.yml ./...
+        working-directory: backend
+
+      - name: run linters for examples
+        run: $GITHUB_WORKSPACE/golangci-lint run --config ${GITHUB_WORKSPACE}/backend/.golangci.yml ./...
+        working-directory: backend/_example/memory_store
+
+      - name: submit coverage
+        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-test-frontend.yml

@@ -0,0 +1,28 @@
+name: test_frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+
+jobs:
+  frontend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions/setup-node@v1
+
+      - run: npm ci
+        working-directory: ./frontend
+
+      - run: npx run-p check lint test
+        working-directory: ./frontend

.gitignore

@@ -19,3 +19,7 @@ debug.test
 remark42
 /bin/
 /backend/var/
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor

.golangci.yml

@@ -0,0 +1,2 @@
+service:
+  golangci-lint-version: 1.19.1

.travis.yml

@@ -1,25 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
-      .
-  - docker rm -f mongo
-  

Dockerfile

@@ -1,17 +1,6 @@
 FROM umputun/baseimage:buildgo-latest as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
 ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
@@ -19,44 +8,29 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
+ARG BACKEND_TEST_TIMEOUT
 
 ADD backend /build/backend
-ADD .git /build/.git
+ADD .git/ /build/backend/.git/
 WORKDIR /build/backend
 
+ENV GOFLAGS="-mod=vendor"
+
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=count -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
-    else echo "skip backend test" ; fi
-
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
-# linters
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-        --enable=ineffassign --enable=varcheck ./... ; \
-    else echo "skip backend linters" ; fi
-
-# submit coverage to coverals if COVERALLS_TOKEN in env
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo "coverall not enabled" ; \
-    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+        golangci-lint run --config ../.golangci.yml ./... ; \
+    else echo "skip backend tests and linter" ; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
-    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
     echo "version=$version" && \
-    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
-
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 FROM node:10.11-alpine as build-frontend-deps
 
@@ -77,16 +51,15 @@ ARG NODE_ENV=production
 COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
 ADD frontend /srv/frontend
 RUN cd /srv/frontend && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p check lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-
-FROM umputun/baseimage:app-latest
+FROM umputun/baseimage:app
 
 WORKDIR /srv
 
-ADD entrypoint.sh /entrypoint.sh
+ADD docker-init.sh /entrypoint.sh
 ADD backend/scripts/backup.sh /usr/local/bin/backup
 ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
@@ -100,5 +73,6 @@ RUN ln -s /srv/remark42 /usr/bin/remark42
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["server"]
-ENTRYPOINT ["/entrypoint.sh"]
+COPY docker-init.sh /srv/init.sh
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -26,72 +26,81 @@ RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
-
 FROM umputun/baseimage:buildgo-latest as build-backend
 
 ARG GITHUB_TOKEN
 ENV SKIP_BACKEND_TEST=true
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD README.md /go/src/github.com/umputun/remark/
-ADD LICENSE /go/src/github.com/umputun/remark/
+RUN go get github.com/rakyll/statik
+
+WORKDIR /build/backend
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+ADD .git/ /build/backend/.git/
+
 COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
-    export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
+    export WEB_ROOT=/build/backend/web && \
     sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
     sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
-    go get -v github.com/rakyll/statik && \
-    statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
-    ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
-    ls -la /go/src/github.com/umputun/remark/backend/web/
+    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
+    ls -la /build/backend/app/rest/api/statik.go && \
+    ls -la /build/backend/web/
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
     if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    echo "runs outside of drone" && version=$(/script/git-rev.sh); \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
+    export GOFLAGS="-mod=vendor" && \
     GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
-    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
 
 RUN \
     if [ -z "$DRONE_TAG" ] ; then \
     echo "runs outside of drone" && tag=""; \
     else tag=_${DRONE_TAG}; fi && \
     apk add --no-cache --update zip && \
-    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
-    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
 # upload to github
-RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "skip upload to github" ; \
-    else \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+#RUN \
+#    if [ -z "$DRONE_TAG" ] ; then \
+#    echo "skip upload to github" ; \
+#    else \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
 RUN ls -la /artifacts/*
 CMD ["sleep", "100"]

Makefile

@@ -20,7 +20,21 @@ deploy:
 	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
 	docker rm -f remark42.bin
 
-.PHONY: bin
+race_test:
+	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,11 +1,14 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# remark42 [![Build Status](https://github.com/umputun/remark/workflows/build/badge.svg)](https://github.com/umputun/remark/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+
+
 
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, GitHub and Yandex
+* Login via email
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus and wordpress
+* Import from Disqus and WordPress
 * Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
@@ -13,7 +16,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
-* Telegram notifications
+* Telegram and email notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
@@ -23,6 +26,16 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
+
+<details><summary>Screenshots</summary>
+
+Comments example:
+![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
+
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
+</details>
+
 
 #
 
@@ -37,6 +50,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
         - [Facebook Auth Provider](#facebook-auth-provider)
+        - [Twitter Auth Provider](#twitter-auth-provider)
         - [Yandex Auth Provider](#yandex-auth-provider)
       - [Initial import from Disqus](#initial-import-from-disqus)
       - [Initial import from WordPress](#initial-import-from-wordpress)
@@ -87,81 +101,123 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default                  | Description                                      |
-| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
-| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                          | secret key, _required_                           |
-| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                          | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
-| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
-| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
-| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
-| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
-| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
-| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
-| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
-| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
-| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
-| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
-| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
-| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
-| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
-| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
-| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                     |
+| ----------------------- | ----------------------- | ------------------------ | ----------------------------------------------- |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_              |
+| secret                  | SECRET                  |                          | secret key, _required_                          |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                           |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `rpc`                |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                          |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                           |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_         |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                     |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                        |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited     |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited       |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited  |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `uri`  |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                 |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                     |
+| avatar.uri              | AVATAR_URI              | `./var/avatars`          | avatar store uri                                |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save     |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, `bolt`             |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                          |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                    |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                      |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                      |
+| image.bolt.file         | IMAGE_BOLT_FILE         | `/var/pictures.db`       | images bolt file location                       |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `2400`                   | width of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `900`                    | height of resized image                         |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                         |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                      |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                          |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                      |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                        |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                    |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                          |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                      |
+| auth.twitter.cid        | AUTH_TWITTER_CID        |                          | Twitter Consumer API Key                        |
+| auth.twitter.csec       | AUTH_TWITTER_CSEC       |                          | Twitter Consumer API Secret key                 |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                          |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                      |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only      |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                          |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                           |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                      |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                   |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                              |
+| auth.email.template     | AUTH_EMAIL_TEMPLATE     | none (predefined)        | custom email message template file              |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (telegram and/or email)    |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                      |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                  |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
+| notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
+| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
+| smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
+| smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
+| smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
+| smtp.password           | SMTP_PASSWORD           |                          | SMTP password                                   |
+| smtp.tls                | SMTP_TLS                |                          | enable TLS for SMTP                             |
+| smtp.timeout            | SMTP_TIMEOUT            | `10s`                    | SMTP TCP connection timeout                     |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le  |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                           |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                           |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                            |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored      |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                            |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited       |
+| votes-ip                | VOTES_IP                | `false`                  | restrict votes from the same ip                 |
+| anon-vote               | ANON_VOTE               | `false`                  | allow voting for anonymous users, require VOTES_IP to be enabled as well |
+| votes-ip-time           | VOTES_IP_TIME           | `5m`                     | same ip vote restriction time, `0s` - unlimited |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                             |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                        |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive   |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_ |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                     |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                 |
+| image-proxy.http2https  |  IMAGE_PROXY_HTTP2HTTPS | `false`                  | enable http->https proxy for images             |
+| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false`            | enable caching external images to current image storage |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                            |
+| simple-view             | SIMPLE_VIEW             | `false`                  | minimized UI with basic info only               |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                 |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                       |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                               |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                 |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                      |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
 * _required_ parameters have to be presented in the environment or provided in command line
 
+##### Deprecated
+
+Following list of command-line options is deprecated and will be removed in 2 minor releases or 1 major release (whichever is closer)
+from the version in which they were deprecated. After remark42 version update, please check startup log once for deprecation warnings to avoid
+trouble with unrecognized command-line options in the future.
+
+<details>
+<summary>deprecated options</summary>
+
+| Command line       | Replacement   | Environment        | Replacement   | Default | Description    | Deprecation version |
+| ------------------ | ------------- | ------------------ | ------------- | ------- | -------------- | ------------------- |
+| auth.email.host    | smtp.host     | AUTH_EMAIL_HOST    | SMTP_HOST     |         | smtp host      | 1.5.0               |
+| auth.email.port    | smtp.port     | AUTH_EMAIL_PORT    | SMTP_PORT     |         | smtp port      | 1.5.0               |
+| auth.email.user    | smtp.username | AUTH_EMAIL_USER    | SMTP_USERNAME |         | smtp user name | 1.5.0               |
+| auth.email.passwd  | smtp.password | AUTH_EMAIL_PASSWD  | SMTP_PASSWORD |         | smtp password  | 1.5.0               |
+| auth.email.tls     | smtp.tls      | AUTH_EMAIL_TLS     | SMTP_TLS      | `false` | enable TLS     | 1.5.0               |
+| auth.email.timeout | smtp.timeout  | AUTH_EMAIL_TIMEOUT | SMTP_TIMEOUT  | `10s`   | smtp timeout   | 1.5.0               |
+| img-proxy          | image-proxy.http2https | IMG_PROXY | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images | 1.5.0 |
+</details>
+
 ##### Required parameters
 
 Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
-2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
+2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.remark42.com`
 3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
 
 The minimal `docker-compose.yml` has to include all required parameters:
@@ -227,6 +283,14 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
 1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
 
+##### Twitter Auth Provider
+
+1.	Create a new twitter application https://developer.twitter.com/en/apps
+1.	Fill **App name**, **Description** and **URL** of your site
+1.	In the field **Callback URLs** enter the correct url of your callback handler e.g.  domain + `/auth/twitter/callback`
+1.	Under **Key and tokens** take note of the **Consumer API Key** and **Consumer API Secret key**. Those will be used as `AUTH_TWITTER_CID` and
+ `AUTH_TWITTER_CSEC`
+
 ##### Yandex Auth Provider
 
 1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
@@ -297,6 +361,39 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
+#### Docker parameters
+
+Two parameters allow to customize docker container on the system level:
+
+- `APP_UID` - sets UID to run remark42 application in container (default=1001)
+- `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
+
+_see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
+
+example of compose:
+
+```yaml
+version: '2'
+
+services:
+    remark42:
+        image: umputun/remark42:latest
+        restart: always
+        container_name: "remark42"
+        environment:
+            - APP_UID=2000                          # runs remark42 app with non-default UID
+            - TIME_ZONE=GTC                         # sets container time to UTC
+
+            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
+            - SECRET=abcd-123456-xyz-$%^&           # secret key
+            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
+            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
+        volumes:
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
+```
+
+
 ### Setup on your website
 
 #### Comments
@@ -308,18 +405,46 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
+    components: ['embed'], // optional param; which components to load. default to ["embed"]
+                           // to load all components define components as ['embed', 'last-comments', 'counter']
+                           // available component are:
+                           //     - 'embed': basic comments widget
+                           //     - 'last-comments': last comments widget, see `Last Comments` section below
+                           //     - 'counter': counter widget, see `Counter` section below
+    url: 'PAGE_URL', // optional param; if it isn't defined
+                     // `window.location.origin + window.location.pathname` will be used,
+                     //
+                     // Note that if you use query parameters as significant part of url
+                     // (the one that actually changes content on page)
+                     // you will have to configure url manually to keep query params, as
+                     // `window.location.origin + window.location.pathname` doesn't contain query params and
+                     // hash. For example default url for `https://example/com/example-post?id=1#hash`
+                     // would be `https://example/com/example-post`.
+                     //
+                     // The problem with query params is that they often contain useless params added by
+                     // various trackers (utm params) and doesn't have defined order, so Remark treats differently
+                     // all this examples:
+                     // https://example.com/?postid=1&date=2007-02-11
+                     // https://example.com/?date=2007-02-11&postid=1
+                     // https://example.com/?date=2007-02-11&postid=1&utm_source=google
+                     //
+                     // If you deal with query parameters make sure you pass only significant part of it
+                     // in well defined order
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -348,19 +473,24 @@ window.REMARK42.changeTheme('light');
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -376,19 +506,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -402,7 +537,7 @@ You can use as many nodes like this as you need to.
 The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
+Also script can use `url` property from `remark_config` object, or `window.location.origin + window.location.pathname` if nothing else is defined.
 
 ## Build from the source
 
@@ -437,12 +572,9 @@ Frontend docker compose config by default skips running backend related tests an
 
 In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
 
-To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
+To run backend - `go run backend/app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
 It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
-`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
-
-To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
-`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
+`HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
 
 ### Frontend development
 
@@ -475,7 +607,7 @@ Developer build running by `webpack-dev-server` supports devtools for [React](ht
 
 #### Frontend guide
 
-Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
 
 ## API
 
@@ -561,7 +693,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
    }{}
 ```
 
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
 * `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
 * `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
   ```go
@@ -590,21 +722,58 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
   ```go
   type Config struct {
-      Version        string   `json:"version"`
-      EditDuration   int      `json:"edit_duration"`
-      MaxCommentSize int      `json:"max_comment_size"`
-      Admins         []string `json:"admins"`
-      AdminEmail     string   `json:"admin_email"`
-      Auth           []string `json:"auth_providers"`
-      LowScore       int      `json:"low_score"`
-      CriticalScore  int      `json:"critical_score"`
-      PositiveScore  bool     `json:"positive_score"`
-      ReadOnlyAge    int      `json:"readonly_age"`
-      MaxImageSize   int      `json:"max_image_size"`
+        Version        string   `json:"version"`
+        EditDuration   int      `json:"edit_duration"`
+        MaxCommentSize int      `json:"max_comment_size"`
+        Admins         []string `json:"admins"`
+        AdminEmail     string   `json:"admin_email"`
+        Auth           []string `json:"auth_providers"`
+        LowScore       int      `json:"low_score"`
+        CriticalScore  int      `json:"critical_score"`
+        PositiveScore  bool     `json:"positive_score"`
+        ReadOnlyAge    int      `json:"readonly_age"`
+        MaxImageSize   int      `json:"max_image_size"`
+        EmojiEnabled   bool     `json:"emoji_enabled"`
   }
   ```
 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
+* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
+
+### Streaming API
+
+Streaming API provide server-sent events for post updates as well as site update
+
+* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
+* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
+
+<details><summary>response example</summary>
+
+```
+data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}
+```
+
+</details>
 
 ### RSS feeds
 
@@ -619,6 +788,17 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
 _returned id should be appended to load image url on caller side_
 
+### Email subscription
+
+* `GET /api/v1/email?site=site-id` - get user's email, _auth required_
+* `POST /api/v1/email/subscribe?site=site-id&address=user@example.org` -  makes confirmation token and sends it to user over email, _auth required_
+
+  Trying to subscribe same email second time will return response code `409 Conflict` and explaining error message.
+* `POST /api/v1/email/confirm?site=site-id&tkn=token` - uses provided token parameter to set email for the user, _auth required_
+
+  Setting email subscribe user for all first-level replies to his messages.
+* `DELETE /api/v1/email?site=siteID` - removes user's email, _auth required_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -631,10 +811,17 @@ _returned id should be appended to load image url on caller side_
       Until     time.Time `json:"time"`
   }
   ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
-* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
+* `GET /api/v1/admin/export?site=site-id&mode=[stream|file]` - export all comments to json stream or gz file.
+* `POST /api/v1/admin/import?site=site-id` - import comments from the backup, uses post body.
+* `POST /api/v1/admin/import/form?site=site-id` - import comments from the backup, user post form.
+* `POST /api/v1/admin/remap?site=site-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
+From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
+export/import chain so make backup first.
+    ```
+    http://oldsite.com* https://newsite.com*
+    http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
+    ```
+* `GET /api/v1/admin/wait?site=site-id` - wait for completion for any async migration ops (import or remap).
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
 * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
 * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.

backend/.golangci.yml

@@ -0,0 +1,22 @@
+run:
+  tests: false
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+linters:
+  enable:
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+  disable-all: true

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,22 @@
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+#ADD . /build/memory_store
+#WORKDIR /build/memory_store
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+
+FROM umputun/baseimage:app-latest
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,13 @@
+# sample store implementation 
+
+`memory_store` illustrates how to make a custom storage plugin for remark42. 
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. docker-compose -f compose-dev-memstore.yml build
+1. docker-compose -f compose-dev-memstore.yml up
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package
+. In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key() (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, errors.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", errors.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, errors.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return errors.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated += 1 // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,563 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store/engine"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.Lock()
+	defer m.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", errors.New("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.Lock()
+	defer m.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, errors.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, errors.New("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.Lock()
+	defer m.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		m.Lock()
+		defer m.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.Lock()
+			defer m.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.Lock()
+	defer m.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return errors.New("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return errors.New("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	return status, errors.Wrapf(err, "failed to set flag %+v", req)
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, errors.New("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID == comment.ID && c.Locator == comment.Locator {
+			c.Text = comment.Text
+			c.Orig = comment.Orig
+			c.Score = comment.Score
+			c.Votes = comment.Votes
+			c.Pin = comment.Pin
+			c.Deleted = comment.Deleted
+			c.User = comment.User
+			comments[i] = c
+			m.posts[comment.Locator.SiteID] = comments
+			return nil
+		}
+	}
+	return errors.New("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,784 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds []string
+	for _, x := range blockedList {
+		blockedIds = append(blockedIds, x.ID)
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		emailSet   = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,47 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: '2'
+
+services:
+
+  remark42:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42"
+    hostname: "remark42"
+    restart: always
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - BACKUP_PATH=/srv/var/backup
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: .
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,16 @@
+module github.com/umputun/remark/memory_store
+
+go 1.12
+
+require (
+	github.com/go-pkgz/jrpc v0.1.0
+	github.com/go-pkgz/lgr v0.6.3
+	github.com/jessevdk/go-flags v1.4.0
+	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.4.0
+	github.com/umputun/remark/backend v1.4.0
+)
+
+replace github.com/umputun/remark/backend => ../../
+
+replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1

backend/_example/memory_store/go.sum

@@ -0,0 +1,330 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
+github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-pkgz/auth v0.9.0/go.mod h1:8Kiold3xA9VUiYZwMyMhwBgiGImZFoL/EmtO9lSIQxo=
+github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
+github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
+github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
+github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
+github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.7.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
+github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a h1:Yu34BogBivvmu7SAzHHaB9nZWH5D1C+z3F1jyIaYZSQ=
+golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
+gopkg.in/oauth2.v3 v3.11.0/go.mod h1:ZdKwOng79bFq17t/ItHGtL8LJ2dW3tmrdJJhHmqS+uc=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/_example/memory_store/main.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/jessevdk/go-flags"
+
+	"github.com/umputun/remark/memory_store/accessor"
+	"github.com/umputun/remark/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+
+	rpcServer := jrpc.Server{
+		API:        opts.API,
+		AuthUser:   opts.AuthUser,
+		AuthPasswd: opts.AuthPasswd,
+		Version:    revision,
+		AppName:    "remark42-memory",
+		Logger:     log.Default(),
+	}
+
+	srv := server.NewRPC(dataStore, adminStore, &rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID: "remark",
+		IDs:    []string{"dev_user"},
+		Email:  "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/rpc.go

@@ -0,0 +1,234 @@
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
+// Note: this file can be used as-is in any custom jrpc plugin
+type RPC struct {
+	*jrpc.Server
+	eng engine.Interface
+	adm admin.Store
+}
+
+// NewRPC makes RPC instance and register handlers
+func NewRPC(e engine.Interface, a admin.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, Server: r}
+	res.addHandlers()
+	return res
+}
+
+func (s *RPC) addHandlers() {
+	// data store handlers
+	s.Group("store", jrpc.HandlersGroup{
+		"create":      s.createHndl,
+		"find":        s.findHndl,
+		"get":         s.getHndl,
+		"update":      s.updateHndl,
+		"count":       s.countHndl,
+		"info":        s.infoHndl,
+		"flag":        s.flagHndl,
+		"list_flags":  s.listFlagsHndl,
+		"user_detail": s.userDetailHndl,
+		"delete":      s.deleteHndl,
+		"close":       s.closeHndl,
+	})
+
+	// admin store handlers
+	s.Group("admin", jrpc.HandlersGroup{
+		"key":     s.admKeyHndl,
+		"admins":  s.admAdminsHndl,
+		"email":   s.admEmailHndl,
+		"enabled": s.admEnabledHndl,
+		"event":   s.admEventHndl,
+	})
+}
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	key, err := s.adm.Key()
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	ps := []interface{}{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/rpc_test.go

@@ -0,0 +1,432 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+
+	"github.com/umputun/remark/memory_store/accessor"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}
+func TestRPC_admKeyHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	_, port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
+	mg := accessor.NewMemData()
+	adm := accessor.NewMemAdminStore("secret")
+	s = NewRPC(mg, adm, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+
+	admRec := accessor.AdminRec{
+		SiteID:  "test-site",
+		IDs:     []string{"id1", "id2"},
+		Email:   "admin@example.com",
+		Enabled: true,
+	}
+	adm.Set("test-site", admRec)
+
+	admRecDisabled := admRec
+	admRecDisabled.Enabled = false
+	adm.Set("test-site-disabled", admRecDisabled)
+
+	port = chooseRandomUnusedPort()
+	go func() {
+		log.Printf("%v", s.Run(port))
+	}()
+
+	waitForHTTPServerStart(port)
+
+	return s, port, func() {
+		require.NoError(t, s.Shutdown())
+	}
+}

backend/app/cmd/avatar.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"path"
-	"time"
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -78,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
 			return nil, err
@@ -93,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
 }
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}

backend/app/cmd/avatar_test.go

@@ -13,32 +13,15 @@ import (
 
 func TestAvatar_Execute(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -46,9 +29,9 @@ func TestAvatar_Execute(t *testing.T) {
 	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test2.db"})
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }

backend/app/cmd/backup_test.go

@@ -25,13 +25,13 @@ func TestBackup_Execute(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	defer os.Remove("/tmp/remark-test.export")
 
 	data, err := ioutil.ReadFile("/tmp/remark-test.export")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
@@ -49,7 +49,7 @@ func TestBackup_ExecuteFailedStatus(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
 }
@@ -68,7 +68,7 @@ func TestBackup_ExecuteFailedWrite(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
 		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,8 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL) //nolint
+	client := http.Client{Timeout: 30 * time.Second}
+	r, err := client.Get(listURL)
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +159,8 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL) //nolint
+		client := http.Client{Timeout: 30 * time.Second}
+		r, err = client.Get(commentsURL)
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}

backend/app/cmd/cleanup_test.go

@@ -68,7 +68,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	posts, err := cmd.postsInRange("20181218", "20181219")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts))
@@ -78,7 +78,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	assert.Equal(t, 3, len(posts))
 
 	_, err = cmd.postsInRange("xxx", "yyy")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestCleanup_listComments(t *testing.T) {
@@ -91,7 +91,7 @@ func TestCleanup_listComments(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	comments, err := cmd.listComments("http://test.com/post1")
 	assert.NoError(t, err)
@@ -118,7 +118,7 @@ func TestCleanup_ExecuteSpam(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
 		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("deleted %+v", cleaned.ids)
@@ -136,7 +136,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("set titles for %+v", titledComments.ids)

backend/app/cmd/cmd.go

@@ -21,6 +21,7 @@ import (
 type CommonOptionsCommander interface {
 	SetCommon(commonOpts CommonOpts)
 	Execute(args []string) error
+	HandleDeprecatedFlags() []DeprecatedFlag
 }
 
 // CommonOpts sets externally from main, shared across all commands
@@ -30,6 +31,13 @@ type CommonOpts struct {
 	Revision     string
 }
 
+// DeprecatedFlag contains information about deprecated option
+type DeprecatedFlag struct {
+	Old           string
+	New           string
+	RemoveVersion string
+}
+
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
@@ -38,6 +46,9 @@ func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
 	c.Revision = commonOpts.Revision
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated and returns their list
+func (c *CommonOpts) HandleDeprecatedFlags() []DeprecatedFlag { return nil }
+
 // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
 type fileParser struct {
 	site string

backend/app/cmd/cmd_test.go

@@ -31,7 +31,7 @@ func TestExport_ParseFileName(t *testing.T) {
 	for i, tt := range tbl {
 		r, err := tt.p.parse(now)
 		if tt.err {
-			assert.NotNil(t, err)
+			assert.Error(t, err)
 			continue
 		}
 		assert.Equal(t, tt.res, r, "check #%d", i)

backend/app/cmd/import.go

@@ -18,7 +18,7 @@ import (
 // ImportCommand set of flags and command for import
 type ImportCommand struct {
 	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
+	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint
 	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
 	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
 	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`

backend/app/cmd/import_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
@@ -21,7 +20,7 @@ func TestImport_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -34,7 +33,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -43,7 +42,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
@@ -61,21 +60,21 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on no such file")
-	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
+	assert.Error(t, err, "fail on no such file")
+	assert.Contains(t, err.Error(), "no such file or directory")
 
 	cmd = ImportCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on connection refused")
-	assert.True(t, strings.Contains(err.Error(), "connection refused"))
+	assert.Error(t, err, "fail on connection refused")
+	assert.Contains(t, err.Error(), "connection refused")
 
 	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		log.Printf("%+v", r)
@@ -87,10 +86,10 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestImport_ExecuteTimeout(t *testing.T) {
@@ -98,7 +97,7 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 		time.Sleep(500 * time.Millisecond)
 		fmt.Fprintln(w, "some response")
@@ -112,8 +111,8 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "deadline exceeded")
 }

backend/app/cmd/remap.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// RemapCommand set of flags and command for change linkage between comments to
+// different urls based on given rules (input file)
+type RemapCommand struct {
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
+	CommonOpts
+}
+
+// Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
+func (rc *RemapCommand) Execute(args []string) error {
+	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	rulesReader, err := os.Open(rc.InputFile)
+	if err != nil {
+		return errors.Wrapf(err, "cant open file %s", rc.InputFile)
+	}
+
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
+	defer cancel()
+	remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
+	req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
+	if err != nil {
+		return errors.Wrapf(err, "can't make remap request for %s", remapURL)
+	}
+	req.SetBasicAuth("admin", rc.AdminPasswd)
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", remapURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "can't get response")
+	}
+
+	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+	return nil
+}

backend/app/cmd/remap_test.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemap_Execute(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		body, err := ioutil.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(202)
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}

backend/app/cmd/restore_test.go

@@ -19,7 +19,7 @@ func TestRestore_Execute(t *testing.T) {
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "native", r.URL.Query().Get("provider"))
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -32,7 +32,7 @@ func TestRestore_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }

backend/app/cmd/server.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -14,16 +15,19 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/dgrijalva/jwt-go"
+	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
+	"github.com/kyokomi/emoji"
 	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/provider"
+	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/mongo"
-	"github.com/go-pkgz/rest/cache"
+	cache "github.com/go-pkgz/lcw"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
@@ -38,31 +42,38 @@ import (
 
 // ServerCommand with command line flags and env
 type ServerCommand struct {
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
-	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
-	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
-	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
-	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Store      StoreGroup      `group:"store" namespace:"store" env-namespace:"STORE"`
+	Avatar     AvatarGroup     `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
+	Cache      CacheGroup      `group:"cache" namespace:"cache" env-namespace:"CACHE"`
+	Admin      AdminGroup      `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
+	Notify     NotifyGroup     `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	SMTP       SmtpGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
+	Image      ImageGroup      `group:"image" namespace:"image" env-namespace:"IMAGE"`
+	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Stream     StreamGroup     `group:"stream" namespace:"stream" env-namespace:"STREAM"`
+	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation  string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles  int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy      bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize  int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView       bool          `long:"simpler-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
 
 	Auth struct {
 		TTL struct {
@@ -73,13 +84,33 @@ type ServerCommand struct {
 		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
 		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
 		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
 		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
 		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Email     struct {
+			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
+			From         string        `long:"from" env:"FROM" description:"from email address"`
+			Subject      string        `long:"subj" env:"SUBJ" default:"remark42 confirmation" description:"email's subject"`
+			ContentType  string        `long:"content-type" env:"CONTENT_TYPE" default:"text/html" description:"content type"`
+			Host         string        `long:"host" env:"HOST" description:"[deprecated, use --smtp.host] SMTP host"`
+			Port         int           `long:"port" env:"PORT" description:"[deprecated, use --smtp.port] SMTP password"`
+			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"[deprecated, use --smtp.password] SMTP port"`
+			SMTPUserName string        `long:"user" env:"USER" description:"[deprecated, use --smtp.username] enable TLS"`
+			TLS          bool          `long:"tls" env:"TLS" description:"[deprecated, use --smtp.tls] SMTP TCP connection timeout"`
+			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"[deprecated, use --smtp.timeout] SMTP TCP connection timeout"`
+			MsgTemplate  string        `long:"template" env:"TEMPLATE" description:"message template file"`
+		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
 }
 
+// ImageProxyGroup defines options group for image proxy
+type ImageProxyGroup struct {
+	HTTP2HTTPS    bool `long:"http2https" env:"HTTP2HTTPS" description:"enable HTTP->HTTPS proxy"`
+	CacheExternal bool `long:"cache-external" env:"CACHE_EXTERNAL" description:"enable caching for external images"`
+}
+
 // AuthGroup defines options group for auth params
 type AuthGroup struct {
 	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
@@ -88,16 +119,17 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"` // nolint
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` // nolint
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -107,25 +139,26 @@ type ImageGroup struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
 	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
-	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
-	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
 }
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"uri" default:"fs"` //nolint
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+	URI    string `long:"uri" env:"URI" default:"./var/avatars" description:"avatar's store URI"`
+	RszLmt int    `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
 }
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` // nolint
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -133,36 +166,45 @@ type CacheGroup struct {
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
-	URL string `long:"url" env:"URL" description:"mongo url"`
-	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"` //nolint
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
+}
+
+// SmtpGroup defines options for SMTP server connection, used in auth and notify modules
+type SmtpGroup struct {
+	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
+	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
+	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Password string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	TLS      bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	TimeOut  time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
 }
 
 // NotifyGroup defines options for notification
 type NotifyGroup struct {
-	Type      string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" default:"none"`
-	QueueSize int    `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
+	Type      []string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" choice:"email" default:"none" env-delim:","` //nolint
+	QueueSize int      `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
 	Telegram  struct {
 		Token   string        `long:"token" env:"TOKEN" description:"telegram token"`
 		Channel string        `long:"chan" env:"CHAN" description:"telegram channel"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"telegram timeout"`
 		API     string        `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
 	} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
+	Email struct {
+		From                string `long:"fromAddress" env:"FROM" description:"from email address"`
+		VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
+	} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 }
 
 // SSLGroup defines options group for server ssl params
 type SSLGroup struct {
-	Type         string `long:"type" env:"TYPE" description:"ssl (auto)support" choice:"none" choice:"static" choice:"auto" default:"none"`
+	Type         string `long:"type" env:"TYPE" description:"ssl (auto) support" choice:"none" choice:"static" choice:"auto" default:"none"` //nolint
 	Port         int    `long:"port" env:"PORT" description:"port number for https server" default:"8443"`
 	Cert         string `long:"cert" env:"CERT" description:"path to cert.pem file"`
 	Key          string `long:"key" env:"KEY" description:"path to key.pem file"`
@@ -170,6 +212,27 @@ type SSLGroup struct {
 	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
 }
 
+// StreamGroup define options for streaming apis
+type StreamGroup struct {
+	RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
+	TimeOut         time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
+	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
+}
+
+// RPCGroup defines options for remote modules (plugins)
+type RPCGroup struct {
+	API          string        `long:"api" env:"API" description:"rpc extension api url"`
+	TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+	AuthUser     string        `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req cache.FlusherRequest)                                        // evict matched records
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -211,6 +274,40 @@ func (s *ServerCommand) Execute(args []string) error {
 	return nil
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated returns their list
+func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
+	// 1.5.0
+	if s.Auth.Email.Host != "" && s.SMTP.Host == "" {
+		s.SMTP.Host = s.Auth.Email.Host
+		result = append(result, DeprecatedFlag{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.Port != 0 && s.SMTP.Port == 0 {
+		s.SMTP.Port = s.Auth.Email.Port
+		result = append(result, DeprecatedFlag{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TLS && !s.SMTP.TLS {
+		s.SMTP.TLS = s.Auth.Email.TLS
+		result = append(result, DeprecatedFlag{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPUserName != "" && s.SMTP.Username == "" {
+		s.SMTP.Username = s.Auth.Email.SMTPUserName
+		result = append(result, DeprecatedFlag{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPPassword != "" && s.SMTP.Password == "" {
+		s.SMTP.Password = s.Auth.Email.SMTPPassword
+		result = append(result, DeprecatedFlag{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TimeOut != 10*time.Second && s.SMTP.TimeOut == 10*time.Second {
+		s.SMTP.TimeOut = s.Auth.Email.TimeOut
+		result = append(result, DeprecatedFlag{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"})
+	}
+	if s.LegacyImageProxy && !s.ImageProxy.HTTP2HTTPS {
+		s.ImageProxy.HTTP2HTTPS = s.LegacyImageProxy
+		result = append(result, DeprecatedFlag{Old: "img-proxy", New: "image-proxy.http2https", RemoveVersion: "1.7.0"})
+	}
+	return result
+}
+
 // newServerApp prepares application and return it with all active parts
 // doesn't start anything
 func (s *ServerCommand) newServerApp() (*serverApp, error) {
@@ -238,9 +335,10 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make pictures store")
 	}
+	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
 
 	dataService := &service.DataStore{
-		Interface:              storeEngine,
+		Engine:                 storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
@@ -250,6 +348,8 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
+	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
+	dataService.RestrictSameIPVotes.Duration = s.DurationVoteIP
 
 	loadingCache, err := s.makeCache()
 	if err != nil {
@@ -270,17 +370,38 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
+		UrlMapperMaker:    migrator.NewUrlMapper,
 		KeyStore:          adminStore,
 	}
 
-	notifyService, err := s.makeNotify(dataService)
+	var emailNotifications bool
+	notifyService, err := s.makeNotify(dataService, authenticator)
+
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "email":
+			emailNotifications = true
+		}
+	}
+
 	if err != nil {
 		log.Printf("[WARN] failed to make notify service, %s", err)
 		notifyService = notify.NopService // disable notifier
+		emailNotifications = false        // email notifications are not available in this case
 	}
 
-	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
-	commentFormatter := store.NewCommentFormatter(imgProxy)
+	imgProxy := &proxy.Image{
+		HTTP2HTTPS:    s.ImageProxy.HTTP2HTTPS,
+		CacheExternal: s.ImageProxy.CacheExternal,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     s.RemarkURL,
+		ImageService:  imageService,
+	}
+	emojiFmt := store.CommentConverterFunc(func(text string) string { return text })
+	if s.EnableEmoji {
+		emojiFmt = func(text string) string { return emoji.Sprint(text) }
+	}
+	commentFormatter := store.NewCommentFormatter(imgProxy, emojiFmt)
 
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
@@ -303,6 +424,15 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
 		ImageService:     imageService,
+		Streamer: &api.Streamer{
+			TimeOut:   s.Stream.TimeOut,
+			Refresh:   s.Stream.RefreshInterval,
+			MaxActive: int32(s.Stream.MaxActive),
+		},
+		EmailNotifications: emailNotifications,
+		EmojiEnabled:       s.EnableEmoji,
+		AnonVote:           s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:         s.SimpleView,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -400,13 +530,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return result, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+	case "rpc":
+		r := &engine.RPC{Client: jrpc.Client{
+			API:        s.Store.RPC.API,
+			Client:     http.Client{Timeout: s.Store.RPC.TimeOut},
+			AuthUser:   s.Store.RPC.AuthUser,
+			AuthPasswd: s.Store.RPC.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
 	}
@@ -422,24 +553,35 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
-	case "mongo":
-		mgServer, err := s.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
 			return nil, err
 		}
 		return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
+	case "uri":
+		return avatar.NewStore(s.Avatar.URI)
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 	switch s.Image.Type {
+	case "bolt":
+		boltImageStore, err := image.NewBoltStorage(
+			s.Image.Bolt.File,
+			s.Image.MaxSize,
+			s.Image.ResizeHeight,
+			s.Image.ResizeWidth,
+			bolt.Options{},
+		)
+		if err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store:    boltImageStore,
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
+		}, nil
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
@@ -453,7 +595,8 @@ func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 				MaxHeight:  s.Image.ResizeHeight,
 				MaxWidth:   s.Image.ResizeWidth,
 			},
-			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
 		}, nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
@@ -469,45 +612,57 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 				s.Admin.Shared.Email = "admin@" + u.Host
 			}
 		}
-		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return nil, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
-		return admin.NewMongoStore(conn, s.SharedSecret), nil
+		return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
+	case "rpc":
+		r := &admin.RPC{Client: jrpc.Client{
+			API:        s.Admin.RPC.API,
+			Client:     http.Client{Timeout: s.Admin.RPC.TimeOut},
+			AuthUser:   s.Admin.RPC.AuthUser,
+			AuthPasswd: s.Admin.RPC.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
 	}
 }
 
-func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
+func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
 	switch s.Cache.Type {
 	case "mem":
-		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
-	// case "mongo":
-	// 	mgServer, err := s.makeMongo()
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "failed to create mongo server")
-	// 	}
-	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
-	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-	// 		cache.MaxKeys(s.Cache.Max.Items))
+		if err != nil {
+			return nil, errors.Wrap(err, "cache backend initialization")
+		}
+		return cache.NewScache(backend), nil
 	case "none":
-		return &cache.Nop{}, nil
+		return cache.NewScache(&cache.Nop{}), nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
-	if s.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
-}
+var msgTemplate = `
+<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
+	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
+		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and paste this text into “token” field on comments page</i></p>
+		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+	</div>
+	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
+</div>
+</body>
+</html>
+`
 
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
@@ -528,12 +683,33 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
 		providers++
 	}
+	if s.Auth.Twitter.CID != "" && s.Auth.Twitter.CSEC != "" {
+		authenticator.AddProvider("twitter", s.Auth.Twitter.CID, s.Auth.Twitter.CSEC)
+		providers++
+	}
+
 	if s.Auth.Dev {
 		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Email.Enable {
+		params := sender.EmailParams{
+			Host:         s.SMTP.Host,
+			Port:         s.SMTP.Port,
+			SMTPUserName: s.SMTP.Username,
+			SMTPPassword: s.SMTP.Password,
+			TimeOut:      s.SMTP.TimeOut,
+			TLS:          s.SMTP.TLS,
+			From:         s.Auth.Email.From,
+			Subject:      s.Auth.Email.Subject,
+			ContentType:  s.Auth.Email.ContentType,
+		}
+		sndr := sender.NewEmailClient(params, log.Default())
+		authenticator.AddVerifProvider("email", s.loadEmailTemplate(), sndr)
+	}
+
 	if s.Auth.Anonymous {
 		log.Print("[INFO] anonymous access enabled")
 		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
@@ -557,20 +733,83 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 	}
 }
 
-func (s *ServerCommand) makeNotify(dataStore *service.DataStore) (*notify.Service, error) {
-	log.Printf("[INFO] make notify, type=%s", s.Notify.Type)
-	switch s.Notify.Type {
-	case "telegram":
-		tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
-			s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create telegram notification destination")
+// loadEmailTemplate trying to get template from opts MsgTemplate and default to embedded
+// if not defined or failed to load
+func (s *ServerCommand) loadEmailTemplate() string {
+	tmpl := msgTemplate
+	if s.Auth.Email.MsgTemplate != "" {
+		log.Printf("[DEBUG] load email template from %s", s.Auth.Email.MsgTemplate)
+		b, err := ioutil.ReadFile(s.Auth.Email.MsgTemplate)
+		if err == nil {
+			tmpl = string(b)
+		} else {
+			log.Printf("[WARN] failed to load email template from %s, %v", s.Auth.Email.MsgTemplate, err)
 		}
-		return notify.NewService(dataStore, s.Notify.QueueSize, tg), nil
-	case "none":
-		return notify.NopService, nil
 	}
-	return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+	return tmpl
+}
+
+func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *auth.Service) (*notify.Service, error) {
+	var notifyService *notify.Service
+	var destinations []notify.Destination
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "telegram":
+			tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
+				s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create telegram notification destination")
+			}
+			destinations = append(destinations, tg)
+		case "email":
+			emailParams := notify.EmailParams{
+				From:                s.Notify.Email.From,
+				VerificationSubject: s.Notify.Email.VerificationSubject,
+				UnsubscribeURL:      s.RemarkURL + "/email/unsubscribe.html",
+				// TODO: uncomment after #560 frontend part is ready and URL is known
+				//SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
+				TokenGenFn: func(userID, email, site string) (string, error) {
+					claims := token.Claims{
+						Handshake: &token.Handshake{ID: userID + "::" + email},
+						StandardClaims: jwt.StandardClaims{
+							Audience:  site,
+							ExpiresAt: time.Now().Add(100 * 365 * 24 * time.Hour).Unix(),
+							NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+							Issuer:    "remark42",
+						},
+					}
+					tkn, err := authenticator.TokenService().Token(claims)
+					if err != nil {
+						return "", errors.Wrapf(err, "failed to make unsubscription token")
+					}
+					return tkn, nil
+				},
+			}
+			smtpParams := notify.SmtpParams{
+				Host:     s.SMTP.Host,
+				Port:     s.SMTP.Port,
+				TLS:      s.SMTP.TLS,
+				Username: s.SMTP.Username,
+				Password: s.SMTP.Password,
+				TimeOut:  s.SMTP.TimeOut,
+			}
+			emailService, err := notify.NewEmail(emailParams, smtpParams)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create email notification destination")
+			}
+			destinations = append(destinations, emailService)
+		case "none":
+			notifyService = notify.NopService
+		default:
+			return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+		}
+	}
+
+	if len(destinations) != 0 {
+		log.Printf("[INFO] make notify, types=%s", s.Notify.Type)
+		notifyService = notify.NewService(dataStore, s.Notify.QueueSize, destinations...)
+	}
+	return notifyService, nil
 }
 
 func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
@@ -619,6 +858,11 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			c.User.SetAdmin(ds.IsAdmin(c.Audience, c.User.ID))
 			c.User.SetBoolAttr("blocked", ds.IsBlocked(c.Audience, c.User.ID))
+			var err error
+			c.User.Email, err = ds.GetUserEmail(c.Audience, c.User.ID)
+			if err != nil {
+				log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
+			}
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
@@ -626,13 +870,18 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			if claims.User == nil {
 				return false
 			}
+			if claims.User.Audience == "" { // reject empty aud, made with old (pre 0.8.x) version of auth package
+				return false
+			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
 		Logger:            log.Default(),
 		RefreshCache:      newAuthRefreshCache(),
+		UseGravatar:       true,
 	})
 	s.addAuthProviders(authenticator)
 	return authenticator

backend/app/cmd/server_test.go

@@ -5,206 +5,157 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/globalsign/mgo"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
-	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 	body, _ = ioutil.ReadAll(resp.Body)
 	t.Log(string(body))
 
-	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+	email, err := app.dataService.AdminStore.Email("")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_AnonMode(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		o.Auth.Anonymous = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
-	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[5].Name(), "anon auth provider")
 
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	// try to login with bad name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
-
-	app.Wait()
-}
-func TestServerApp_WithMongo(t *testing.T) {
-
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
-
-	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
-	// prepare options
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
-		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
-	require.Nil(t, err)
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
-
-	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
-
-	defer func() {
-		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, e)
-		conn := mongo.NewConnection(s, "test_remark", "")
-		_ = conn.WithDB(func(dbase *mgo.Database) error {
-			assert.NoError(t, dbase.DropDatabase())
-			return nil
-		})
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://localhost:18443", SharedSecret: "123456"})
+	sslPort := chooseRandomUnusedPort()
+	opts.SetCommon(CommonOpts{RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort), SharedSecret: "123456"})
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=" + strconv.Itoa(port), "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
 		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
 		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
-		"--ssl.port=18443", "--image.fs.path=/tmp"})
-	require.Nil(t, err)
+		"--ssl.port=" + strconv.Itoa(sslPort), "--image.fs.path=/tmp"})
+	require.NoError(t, err)
 
 	// create app
 	app, err := opts.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(1 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -219,21 +170,58 @@ func TestServerApp_WithSSL(t *testing.T) {
 	}
 
 	// check http to https redirect response
-	resp, err := client.Get("http://localhost:18080/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:18443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	// check https server
-	resp, err = client.Get("https://localhost:18443/ping")
-	require.Nil(t, err)
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
+	app.Wait()
+}
+
+func TestServerApp_WithRemote(t *testing.T) {
+
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
+		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+	require.NoError(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+	// create app
+	app, err := opts.newServerApp()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { _ = app.run(ctx) }()
+	waitForHTTPServerStart(port)
+
+	// send ping
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	cancel()
 	app.Wait()
 }
 
@@ -245,7 +233,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 	// RO bolt location
 	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -255,7 +243,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -265,7 +253,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
@@ -274,7 +262,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
-	assert.NotNil(t, err, "blah is invalid type")
+	assert.Error(t, err, "blah is invalid type")
 
 	opts.Store.Type = "blah"
 	_, err = opts.newServerApp()
@@ -283,38 +271,86 @@ func TestServerApp_Failed(t *testing.T) {
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = chooseRandomUnusedPort()
 		return o
 	})
+	time.AfterFunc(100*time.Millisecond, func() {
+		cancel()
+	})
 	st := time.Now()
 	err := app.run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err)
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100msec")
 	app.Wait()
 }
 
 func TestServerApp_MainSignal(t *testing.T) {
 
+	done := make(chan struct{})
 	go func() {
+		<-done
 		time.Sleep(250 * time.Millisecond)
 		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}()
-	st := time.Now()
 
 	s := ServerCommand{}
 	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	p := flags.NewParser(&s, flags.Default)
+	port := chooseRandomUnusedPort()
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=" + strconv.Itoa(port), "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	st := time.Now()
+	close(done)
 	err = s.Execute(args)
-	assert.NoError(t, err, "execute failed")
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err, "execute should be without errors")
+	assert.True(t, time.Since(st).Seconds() < 5, "should take under five sec", time.Since(st).Seconds())
+}
+
+func TestServerApp_DeprecatedArgs(t *testing.T) {
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{
+		"test",
+		"--auth.email.host=smtp.example.org",
+		"--auth.email.port=666",
+		"--auth.email.tls",
+		"--auth.email.user=test_user",
+		"--auth.email.passwd=test_password",
+		"--auth.email.timeout=15s",
+	}
+	assert.Empty(t, s.SMTP.Host)
+	assert.Empty(t, s.SMTP.Port)
+	assert.Empty(t, s.SMTP.TLS)
+	assert.Empty(t, s.SMTP.Username)
+	assert.Empty(t, s.SMTP.Password)
+	assert.Empty(t, s.SMTP.TimeOut)
+	_, err := p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlags := s.HandleDeprecatedFlags()
+	assert.ElementsMatch(t,
+		[]DeprecatedFlag{
+			{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"},
+		},
+		deprecatedFlags)
+	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
+	assert.Equal(t, 666, s.SMTP.Port)
+	assert.Equal(t, true, s.SMTP.TLS)
+	assert.Equal(t, "test_user", s.SMTP.Username)
+	assert.Equal(t, "test_password", s.SMTP.Password)
+	assert.Equal(t, 15*time.Second, s.SMTP.TimeOut)
 }
 
 func Test_ACMEEmail(t *testing.T) {
@@ -323,9 +359,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	args := []string{"--ssl.type=auto"}
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err := cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -333,9 +369,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--ssl.acme-email=adminname@adminhost.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "adminname@adminhost.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -343,9 +379,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared", "--admin.shared.email=superadmin@admin.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "superadmin@admin.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -353,20 +389,21 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// make a token for user dev
 	tkService := app.restSrv.Authenticator.TokenService()
@@ -389,51 +426,122 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
+	// add comment with no-aud claim
+	claimsNoAud := claims
+	claimsNoAud.Audience = ""
+	tkNoAud, err := tkService.Token(claimsNoAud)
+	require.NoError(t, err)
+	t.Logf("no-aud claims: %s", tkNoAud)
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/", 
+"site": "remark"}}`))
+	require.NoError(t, err)
+	req.Header.Set("X-JWT", tkNoAud)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
+
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
-	assert.Nil(t, e)
+	req, err = http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, e = client.Do(req)
-	require.Nil(t, e)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Log(string(b))
 
-	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
-
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
+		"blocked user can't post, \n"+tk+"\n"+string(body))
 
+	cancel()
 	app.Wait()
 }
 
-func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
+func TestServer_loadEmailTemplate(t *testing.T) {
+	cmd := ServerCommand{}
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
+	r := cmd.loadEmailTemplate()
+	assert.Equal(t, "The token is {{.Token}}", r)
+
+	cmd.Auth.Email.MsgTemplate = ""
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+
+	cmd.Auth.Email.MsgTemplate = "bad-file"
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}
+
+func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
 	cmd := ServerCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "secret"})
 
 	// prepare options
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
@@ -441,23 +549,27 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
 	cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
+	cmd.Auth.Email.Enable = true
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
 	cmd.BackupLocation = "/tmp"
-	cmd.Notify.Type = "telegram"
-	cmd.Notify.Telegram.API = "http://127.0.0.1:12340/"
-	cmd.Notify.Telegram.Token = "blah"
+	cmd.Notify.Type = []string{"email"}
+	cmd.Notify.Email.From = "from@example.org"
+	cmd.Notify.Email.VerificationSubject = "test verification email subject"
+	cmd.SMTP.Host = "127.0.0.1"
+	cmd.SMTP.Port = 25
+	cmd.SMTP.Username = "test_user"
+	cmd.SMTP.Password = "test_password"
+	cmd.SMTP.TimeOut = time.Second
+	cmd.UpdateLimit = 10
 	cmd = fn(cmd)
 
 	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
 
 	// create app
 	app, err := cmd.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	return app, ctx
+	rand.Seed(time.Now().UnixNano())
+	return app, ctx, cancel
 }

backend/app/cmd/testdata/email.tmpl

@@ -0,0 +1 @@
+The token is {{.Token}}

backend/app/cmd/testdata/remap_urls.txt

@@ -0,0 +1,2 @@
+http://oldsite.com* https://newsite.com*
+http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )
@@ -21,6 +21,7 @@ type Opts struct {
 	RestoreCmd cmd.RestoreCommand `command:"restore"`
 	AvatarCmd  cmd.AvatarCommand  `command:"avatar"`
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
 	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
@@ -44,6 +45,10 @@ func main() {
 			SharedSecret: opts.SharedSecret,
 			Revision:     revision,
 		})
+		for _, entry := range c.HandleDeprecatedFlags() {
+			log.Printf("[WARN] --%s is deprecated and will be removed in v%s, please use --%s instead",
+				entry.Old, entry.RemoveVersion, entry.New)
+		}
 		err := c.Execute(args)
 		if err != nil {
 			log.Printf("[ERROR] failed with %+v", err)

backend/app/main_test.go

@@ -1,19 +1,19 @@
 package main
 
 import (
-	"context"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/repeater"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,45 +24,37 @@ func Test_Main(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
+	port := chooseRandomUnusedPort()
 	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
-		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+		"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
+	done := make(chan struct{})
 	go func() {
-		time.Sleep(5000 * time.Millisecond)
-		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, e)
+		<-done
+		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.NoError(t, err)
 	}()
 
-	wg := sync.WaitGroup{}
-	wg.Add(1)
+	finished := make(chan struct{})
 	go func() {
-		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
-		wg.Done()
+		close(finished)
 	}()
 
-	var passed bool
-	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
-		resp, e := http.Get("http://localhost:18222/api/v1/ping")
-		if e != nil {
-			t.Logf("%+v", e)
-			return e
-		}
-		require.Nil(t, e)
-		defer resp.Body.Close()
-		assert.Equal(t, 200, resp.StatusCode)
-		body, e := ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
-		assert.Equal(t, "pong", string(body))
-		passed = true
-		return nil
-	})
+	// defer cleanup because require check below can fail
+	defer func() {
+		close(done)
+		<-finished
+	}()
 
+	waitForHTTPServerStart(port)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, true, passed, "at least on ping passed")
-
-	wg.Wait()
+	assert.Equal(t, "pong", string(body))
 }
 
 func TestGetDump(t *testing.T) {
@@ -72,3 +64,26 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
 	log.Printf("\n dump: %s", dump)
 }
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 10 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 100; i++ {
+		time.Sleep(time.Millisecond * 100)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}

backend/app/migrator/backup_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBackup_RemoveOldBackupFiles(t *testing.T) {
@@ -21,17 +22,17 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 	}
 	fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
 	err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
 	bk.removeOldBackupFiles()
 	ff, err := ioutil.ReadDir(loc)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
 	assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
 	assert.Equal(t, "backup-site1-20171209.gz", ff[1].Name())
 	assert.Equal(t, "backup-site1-20171210.gz", ff[2].Name())

backend/app/migrator/disqus_test.go

@@ -19,16 +19,17 @@ import (
 func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	d := Disqus{DataStore: &dataStore}
 	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(last), "4 comments imported")
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(last), "4 comments imported")
 
 	c := last[len(last)-1] // last reverses, get first one
 	assert.True(t, strings.HasPrefix(c.Text, "<p>The quick brown fox"))
@@ -40,11 +41,11 @@ func TestDisqus_Import(t *testing.T) {
 	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
 
 	posts, err := dataStore.List("test", 0, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts), "2 posts")
 
 	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, count)
 }
 
@@ -56,7 +57,7 @@ func TestDisqus_Convert(t *testing.T) {
 	for comment := range ch {
 		res = append(res, comment)
 	}
-	assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
+	require.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
 
 	exp0 := store.Comment{
 		ID: "299619020",

backend/app/migrator/mapper.go

@@ -0,0 +1,74 @@
+package migrator
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+// UrlMapper implements Mapper interface
+type UrlMapper struct {
+	rules map[string]string
+}
+
+// NewUrlMapper reads rules from given reader and returns initialised UrlMapper
+// if given rules are valid.
+func NewUrlMapper(reader io.Reader) (Mapper, error) {
+	u := &UrlMapper{}
+	if err := u.loadRules(reader); err != nil {
+		return u, err
+	}
+	return u, nil
+}
+
+// loadRules loads url-mapping rules from reader to mapper.
+// Rules must be a text consists of rows separated by \n.
+// Each row holds from-url and to-url separated by space.
+// If urls end with asterisk (*) it means try to match by prefix.
+// Example:
+// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
+// https://www.myblog.com/* https://myblog.com/*
+func (u *UrlMapper) loadRules(reader io.Reader) error {
+	data, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	rulesText := strings.TrimSpace(string(data))
+
+	u.rules = make(map[string]string)
+
+	for _, row := range strings.Split(rulesText, "\n") {
+		row = strings.TrimSpace(row)
+		urls := strings.Split(row, " ")
+		if len(urls) != 2 {
+			return errors.New("bad row " + row)
+		}
+
+		from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
+		u.rules[from] = to
+	}
+	return nil
+}
+
+// URL maps given url to another url according loaded url-rules.
+// If not matched returns given url.
+func (u *UrlMapper) URL(url string) string {
+	if newUrl, ok := u.rules[url]; ok {
+		return newUrl
+	}
+	// try to match by prefix
+	for oldUrl, newUrl := range u.rules {
+		if !strings.HasSuffix(oldUrl, "*") {
+			continue
+		}
+		oldUrl = strings.TrimSuffix(oldUrl, "*")
+		newUrl = strings.TrimSuffix(newUrl, "*")
+		if strings.HasPrefix(url, oldUrl) {
+			return newUrl + strings.TrimPrefix(url, oldUrl)
+		}
+	}
+	// search failed, return given url
+	return url
+}

backend/app/migrator/mapper_test.go

@@ -0,0 +1,90 @@
+package migrator
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUrlMapper_URL(t *testing.T) {
+	// want remap urls from https://radio-t.com to https://www.radio-t.com
+	// also map individual urls
+	rules := strings.NewReader(`
+https://radio-t.com* https://www.radio-t.com*
+https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
+https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
+`)
+
+	mapper, err := NewUrlMapper(rules)
+	assert.NoError(t, err)
+
+	// if url not matched mapper should return given url
+	assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
+	assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
+	// check strict matching
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
+	// check pattern matching (by prefix)
+	assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
+
+	// want remap from http to https
+	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
+	mapper, err = NewUrlMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
+
+	// want remap from http to https by pattern
+	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
+	mapper, err = NewUrlMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+}
+
+func TestUrlMapper_New(t *testing.T) {
+	cases := []struct {
+		rules       string
+		expectError bool
+	}{
+		// bad input, expect error
+		{
+			rules:       "https://radio-t.com ",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com\n https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com   \n https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+
+		// valid input, no error
+		{
+			rules: "https://radio-t.com* https://www.radio-t.com*",
+		},
+		{
+			rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
+		},
+		{
+			rules: "https://any.com/p/111 https://any.com/p/222   \n https://any.com/p/333 https://any.com/p/222   \n",
+		},
+	}
+	for _, c := range cases {
+		_, err := NewUrlMapper(strings.NewReader(c.rules))
+		if c.expectError {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}

backend/app/migrator/migrator.go

@@ -24,10 +24,20 @@ type Exporter interface {
 	Export(w io.Writer, siteID string) (int, error)
 }
 
+// Mapper defines interface to convert data in import procedure
+type Mapper interface {
+	URL(url string) string
+}
+
+// MapperMaker defines function that reads rules from reader and
+// returns new Mapper with loaded rules. If rules are not valid
+// it returns error.
+type MapperMaker func(reader io.Reader) (Mapper, error)
+
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
@@ -42,6 +52,8 @@ type ImportParams struct {
 	SiteID    string
 }
 
+var adminUser = store.User{Admin: true}
+
 // ImportComments imports from given provider format and saves to store
 func ImportComments(p ImportParams) (int, error) {
 	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)

backend/app/migrator/migrator_test.go

@@ -4,11 +4,13 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
@@ -21,22 +23,23 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
 		SiteID:    "test",
 		Provider:  "disqus",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
 
@@ -47,22 +50,23 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
 		SiteID:    "test",
 		Provider:  "wordpress",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
@@ -76,11 +80,12 @@ func TestMigrator_ImportNative(t *testing.T) {
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
 	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -88,19 +93,20 @@ func TestMigrator_ImportNative(t *testing.T) {
 		SiteID:    "radio-t",
 		Provider:  "native",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	last, err := dataStore.Last("radio-t", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
 
 func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b}
+	defer dataStore.Close()
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -49,7 +49,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	commentsCount := 0
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
-		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
 		if e != nil {
 			return commentsCount, e
 		}
@@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
 	return nil
 }
 
+// WithMapper wraps reader with url-mapper.
+func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			log.Printf("[DEBUG] finish write to pipe with %+v", err)
+			if e := w.Close(); e != nil {
+				log.Printf("[WARN] failed close pipe writer with %+v", e)
+			}
+		}()
+
+		// decode from reader and encode to pipe writer
+		dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
+
+		m := meta{}
+		if err = dec.Decode(&m); err != nil {
+			return
+		}
+		for i := range m.Posts {
+			m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
+		}
+		if err = enc.Encode(m); err != nil {
+			return
+		}
+
+		for {
+			comment := store.Comment{}
+			if err = dec.Decode(&comment); err != nil {
+				return
+			}
+			comment.Locator.URL = mapper.URL(comment.Locator.URL)
+			if err = enc.Encode(comment); err != nil {
+				return
+			}
+		}
+	}()
+
+	return r
+}
+
 // Import comments from json strings produced by Remark.Export
 func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
-
 	m := meta{}
 	dec := json.NewDecoder(reader)
 	if err = dec.Decode(&m); err != nil {

backend/app/migrator/native_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
@@ -20,11 +21,9 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
-
 func TestNative_Export(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // write 2 comments
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
 	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
@@ -32,7 +31,7 @@ func TestNative_Export(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
@@ -40,25 +39,25 @@ func TestNative_Export(t *testing.T) {
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
-	meta := struct {
+	m := struct {
 		Version int                    `json:"version"`
 		Users   []service.UserMetaData `json:"users"`
 		Posts   []service.PostMetaData `json:"posts"`
 	}{}
 
-	require.NoError(t, dec.Decode(&meta), "decode meta")
+	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(meta.Users))
-	assert.Equal(t, "user1", meta.Users[0].ID)
-	assert.Equal(t, false, meta.Users[0].Blocked.Status)
-	assert.Equal(t, true, meta.Users[0].Verified)
-	assert.Equal(t, "user2", meta.Users[1].ID)
-	assert.Equal(t, true, meta.Users[1].Blocked.Status)
-	assert.Equal(t, false, meta.Users[1].Verified)
+	require.Equal(t, 2, len(m.Users))
+	assert.Equal(t, "user1", m.Users[0].ID)
+	assert.Equal(t, false, m.Users[0].Blocked.Status)
+	assert.Equal(t, true, m.Users[0].Verified)
+	assert.Equal(t, "user2", m.Users[1].ID)
+	assert.Equal(t, true, m.Users[1].Blocked.Status)
+	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(meta.Posts))
-	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
-	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+	require.Equal(t, 1, len(m.Posts))
+	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
+	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
 	comments := [3]store.Comment{}
 
@@ -70,21 +69,22 @@ func TestNative_Export(t *testing.T) {
 }
 
 func TestNative_Import(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
 	assert.Equal(t, "1234", comments[0].ParentID)
 	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
@@ -100,49 +100,93 @@ func TestNative_Import(t *testing.T) {
 	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
 }
 
+func TestNative_ImportWithMapper(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
+	// want to remap comments to https://rdt.c
+	rules := `https://radio-t.com* https://rdt.c*`
+	mapper, err := NewUrlMapper(strings.NewReader(rules))
+	assert.NoError(t, err)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	mappedReader := WithMapper(strings.NewReader(inp), mapper)
+
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
+	size, err := r.Import(mappedReader, "radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
 func TestNative_ImportWrongVersion(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
 	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
 	assert.Equal(t, 0, size)
 
 }
 func TestNative_ImportManyWithError(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
 
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
-	for i := 0; i < 1200; i++ {
+	for i := 0; i < 100; i++ {
 		buf.WriteString(fmt.Sprintf(goodRec, i))
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
+	assert.Equal(t, 100, n)
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
+	assert.NoError(t, err)
+	assert.Equal(t, 100, len(comments))
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
+func prep(t *testing.T) (*service.DataStore, func()) {
+
+	testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
@@ -152,7 +196,7 @@ func prep(t *testing.T) *service.DataStore {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
@@ -160,7 +204,10 @@ func prep(t *testing.T) *service.DataStore {
 		User:    store.User{ID: "user2", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	return b
+	return b, func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
 }

backend/app/migrator/wordpress.go

@@ -143,7 +143,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -17,19 +18,20 @@ import (
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
-	assert.Nil(t, err, "create store")
+	assert.NoError(t, err, "create store")
 
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last(siteID, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
+	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
 
 	c := last[0]
 	assert.Equal(t, "14", c.ID)
@@ -42,14 +44,14 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
 
 	posts, err := dataStore.List(siteID, 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(posts))
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(posts))
 
 	p := posts[0]
 	assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
 
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
 }
 
@@ -61,7 +63,7 @@ func TestWordPress_Convert(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
+	require.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
 
 	exp1 := store.Comment{
 		ID: "13",
@@ -88,7 +90,7 @@ func TestWP_Convert_MD(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported")
+	require.Equal(t, 3, len(comments), "3 comments exported")
 
 	assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
 

backend/app/notify/email.go

@@ -0,0 +1,456 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"mime/quotedprintable"
+	"net"
+	"net/smtp"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
+	"github.com/pkg/errors"
+)
+
+// EmailParams contain settings for email notifications
+type EmailParams struct {
+	From                 string // from email address
+	MsgTemplate          string // request message template
+	VerificationSubject  string // verification message subject
+	VerificationTemplate string // verification message template
+	SubscribeURL         string // full subscribe handler URL
+	UnsubscribeURL       string // full unsubscribe handler URL
+
+	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
+}
+
+// SmtpParams contain settings for smtp server connection
+type SmtpParams struct {
+	Host     string        // SMTP host
+	Port     int           // SMTP port
+	TLS      bool          // TLS auth
+	Username string        // user name
+	Password string        // password
+	TimeOut  time.Duration // TCP connection timeout
+}
+
+// Email implements notify.Destination for email
+type Email struct {
+	EmailParams
+	SmtpParams
+
+	smtp       smtpClientCreator
+	msgTmpl    *template.Template // parsed request message template
+	verifyTmpl *template.Template // parsed verification message template
+}
+
+// default email client implementation
+type emailClient struct{ smtpClientCreator }
+
+// smtpClient interface defines subset of net/smtp used by email client
+type smtpClient interface {
+	Mail(string) error
+	Auth(smtp.Auth) error
+	Rcpt(string) error
+	Data() (io.WriteCloser, error)
+	Quit() error
+	Close() error
+}
+
+// smtpClientCreator interface defines function for creating new smtpClients
+type smtpClientCreator interface {
+	Create(SmtpParams) (smtpClient, error)
+}
+
+type emailMessage struct {
+	from    string
+	to      string
+	message string
+}
+
+// msgTmplData store data for message from request template execution
+type msgTmplData struct {
+	UserName          string
+	UserPicture       string
+	CommentText       string
+	CommentLink       string
+	CommentDate       time.Time
+	ParentUserName    string
+	ParentUserPicture string
+	ParentCommentText string
+	ParentCommentLink string
+	ParentCommentDate time.Time
+	PostTitle         string
+	Email             string
+	UnsubscribeLink   string
+}
+
+// verifyTmplData store data for verification message template execution
+type verifyTmplData struct {
+	User         string
+	Token        string
+	Email        string
+	Site         string
+	SubscribeURL string
+}
+
+const (
+	defaultVerificationSubject = "Email verification"
+	defaultEmailTimeout        = 10 * time.Second
+	defaultEmailTemplate       = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+	<style type="text/css">
+		img {
+			max-width: 100%;
+			max-height: 250px;
+			margin: 5px 0;
+			display: block;
+			color: #000;
+		}
+		a {
+			text-decoration: none;
+			color: #0aa;
+		}
+		p {
+			margin: 0 0 12px;
+		}
+	</style>
+</head>
+<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+<body>
+	<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
+		<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
+		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+		<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
+			<div>
+				<div style="margin-bottom: 12px; line-height: 24px;">
+					<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+					<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
+					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
+					<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
+				</div>
+				<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
+					{{.ParentCommentText}}
+				</div>
+			</div>
+			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px; line-height: 24px;">
+				<div style="margin-bottom: 8px;">
+					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; float: left; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+					<div style="float: left; font-size: 14px; font-weight: bold; color: #777">
+						{{.UserName}}
+					</div>
+					<div style="color: #999; font-size: 14px; margin: 0 8px; float: left;">
+						{{.CommentDate.Format "02.01.2006 at 15:04"}}
+					</div>
+					<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
+				</div>
+				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">
+					{{.CommentText}}
+				</div>
+			</div>
+		</div>
+		<div style="text-align: center; font-size: 14px; margin-top: 32px;">
+			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a> for {{.ParentUserName}}</i>
+			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;">
+				<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
+				<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
+				<div style="opacity: 0;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
+			</div>
+		</div>
+	</div>
+</body>
+</html>
+`
+	defaultEmailVerificationTemplate = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+	<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+	<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
+		<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+		<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
+		{{if .SubscribeURL}}
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
+		{{ end }}
+		<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+			<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
+			<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
+			<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+		</div>
+		<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
+	</div>
+</body>
+</html>
+`
+)
+
+// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
+func NewEmail(emailParams EmailParams, smtpParams SmtpParams) (*Email, error) {
+	// set up Email emailParams
+	res := Email{EmailParams: emailParams}
+	if res.MsgTemplate == "" {
+		res.MsgTemplate = defaultEmailTemplate
+	}
+	if res.VerificationTemplate == "" {
+		res.VerificationTemplate = defaultEmailVerificationTemplate
+	}
+	if res.VerificationSubject == "" {
+		res.VerificationSubject = defaultVerificationSubject
+	}
+
+	// set up SMTP emailParams
+	res.smtp = &emailClient{}
+	res.SmtpParams = smtpParams
+	if res.TimeOut <= 0 {
+		res.TimeOut = defaultEmailTimeout
+	}
+
+	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
+		res.Host, res.Username, res.TimeOut)
+
+	// initialise templates
+	var err error
+	if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse message template")
+	}
+	if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse verification template")
+	}
+	return &res, err
+}
+
+// Send email about reply to Request.Email if it's set, otherwise do nothing and return nil, thread safe
+// do not returns sending error, only following:
+// 1. (likely impossible) template execution error from email message creation from Request
+// 2. message dropped without sending in case of closed ctx
+func (e *Email) Send(ctx context.Context, req Request) (err error) {
+	if req.Email == "" {
+		// this means we can't send this request via Email
+		return nil
+	}
+	select {
+	case <-ctx.Done():
+		return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
+	default:
+	}
+	var msg string
+
+	if req.Verification.Token != "" {
+		log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
+		msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+		if err != nil {
+			return err
+		}
+	}
+
+	if req.Comment.ID != "" {
+		if req.parent.User.ID == req.Comment.User.ID {
+			// don't send anything if if user replied to their own comment
+			return nil
+		}
+		log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
+		msg, err = e.buildMessageFromRequest(req)
+		if err != nil {
+			return err
+		}
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.sendMessage(emailMessage{from: e.From, to: req.Email, message: msg})
+		})
+}
+
+// buildVerificationMessage generates verification email message based on given input
+func (e *Email) buildVerificationMessage(user, email, token, site string) (string, error) {
+	subject := e.VerificationSubject
+	msg := bytes.Buffer{}
+	err := e.verifyTmpl.Execute(&msg, verifyTmplData{
+		User:         user,
+		Token:        token,
+		Email:        email,
+		Site:         site,
+		SubscribeURL: e.SubscribeURL,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build verification message")
+	}
+	return e.buildMessage(subject, msg.String(), email, "text/html", "")
+}
+
+// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
+func (e *Email) buildMessageFromRequest(req Request) (string, error) {
+	subject := "New reply to your comment"
+	if req.Comment.PostTitle != "" {
+		subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
+	}
+	token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if err != nil {
+		return "", errors.Wrapf(err, "error creating token for unsubscribe link")
+	}
+	commentUrlPrefix := req.Comment.Locator.URL + uiNav
+	msg := bytes.Buffer{}
+	err = e.msgTmpl.Execute(&msg, msgTmplData{
+		UserName:          req.Comment.User.Name,
+		UserPicture:       req.Comment.User.Picture,
+		CommentText:       req.Comment.Text,
+		CommentLink:       commentUrlPrefix + req.Comment.ID,
+		CommentDate:       req.Comment.Timestamp,
+		ParentUserName:    req.parent.User.Name,
+		ParentUserPicture: req.parent.User.Picture,
+		ParentCommentText: req.parent.Text,
+		ParentCommentLink: commentUrlPrefix + req.parent.ID,
+		ParentCommentDate: req.parent.Timestamp,
+		PostTitle:         req.Comment.PostTitle,
+		Email:             req.Email,
+		UnsubscribeLink:   unsubscribeLink,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build comment reply message")
+	}
+	return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
+}
+
+// buildMessage generates email message to send using net/smtp.Data()
+func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink string) (message string, err error) {
+	addHeader := func(msg, h, v string) string {
+		msg += fmt.Sprintf("%s: %s\n", h, v)
+		return msg
+	}
+	message = addHeader(message, "From", e.From)
+	message = addHeader(message, "To", to)
+	message = addHeader(message, "Subject", subject)
+	message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
+
+	if contentType != "" {
+		message = addHeader(message, "MIME-version", "1.0")
+		message = addHeader(message, "Content-Type", contentType+`; charset="UTF-8"`)
+	}
+
+	if unsubscribeLink != "" {
+		// https://support.google.com/mail/answer/81126 -> "Include option to unsubscribe"
+		message = addHeader(message, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
+		message = addHeader(message, "List-Unsubscribe", "<"+unsubscribeLink+">")
+	}
+
+	message = addHeader(message, "Date", time.Now().Format(time.RFC1123Z))
+
+	buff := &bytes.Buffer{}
+	qp := quotedprintable.NewWriter(buff)
+	if _, err := qp.Write([]byte(body)); err != nil {
+		return "", err
+	}
+	defer qp.Close()
+	m := buff.String()
+	message += "\n" + m
+	return message, nil
+}
+
+// sendMessage sends messages to server in a new connection, closing the connection after finishing.
+// Thread safe.
+func (e *Email) sendMessage(m emailMessage) error {
+	if e.smtp == nil {
+		return errors.New("sendMessage called without smtpClient set")
+	}
+	smtpClient, err := e.smtp.Create(e.SmtpParams)
+	if err != nil {
+		return errors.Wrap(err, "failed to make smtp Create")
+	}
+
+	defer func() {
+		if err := smtpClient.Quit(); err != nil {
+			log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
+			if err := smtpClient.Close(); err != nil {
+				log.Printf("[WARN] can't close smtp connection, %v", err)
+			}
+		}
+	}()
+
+	if err := smtpClient.Mail(m.from); err != nil {
+		return errors.Wrapf(err, "bad from address %q", m.from)
+	}
+	if err := smtpClient.Rcpt(m.to); err != nil {
+		return errors.Wrapf(err, "bad to address %q", m.to)
+	}
+
+	writer, err := smtpClient.Data()
+	if err != nil {
+		return errors.Wrap(err, "can't make email writer")
+	}
+
+	defer func() {
+		if err = writer.Close(); err != nil {
+			log.Printf("[WARN] can't close smtp body writer, %v", err)
+		}
+	}()
+
+	buf := bytes.NewBufferString(m.message)
+	if _, err = buf.WriteTo(writer); err != nil {
+		return errors.Wrapf(err, "failed to send email body to %q", m.to)
+	}
+
+	return nil
+}
+
+// String representation of Email object
+func (e *Email) String() string {
+	return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
+}
+
+// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SmtpParams
+// and returns pointer to it. Thread safe.
+func (s *emailClient) Create(params SmtpParams) (smtpClient, error) {
+	authenticate := func(c *smtp.Client) error {
+		if params.Username == "" || params.Password == "" {
+			return nil
+		}
+		auth := smtp.PlainAuth("", params.Username, params.Password, params.Host)
+		if err := c.Auth(auth); err != nil {
+			return errors.Wrapf(err, "failed to auth to smtp %s:%d", params.Host, params.Port)
+		}
+		return nil
+	}
+
+	var c *smtp.Client
+	srvAddress := fmt.Sprintf("%s:%d", params.Host, params.Port)
+	if params.TLS {
+		tlsConf := &tls.Config{
+			InsecureSkipVerify: false,
+			ServerName:         params.Host,
+		}
+		conn, err := tls.Dial("tcp", srvAddress, tlsConf)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to dial smtp tls to %s", srvAddress)
+		}
+		if c, err = smtp.NewClient(conn, params.Host); err != nil {
+			return nil, errors.Wrapf(err, "failed to make smtp client for %s", srvAddress)
+		}
+		return c, authenticate(c)
+	}
+
+	conn, err := net.DialTimeout("tcp", srvAddress, params.TimeOut)
+	if err != nil {
+		return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
+	}
+
+	c, err = smtp.NewClient(conn, srvAddress)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to dial")
+	}
+
+	return c, authenticate(c)
+}

backend/app/notify/email_test.go

@@ -0,0 +1,359 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/smtp"
+	"sync"
+	"testing"
+	"text/template"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestEmailNew(t *testing.T) {
+	var testSet = []struct {
+		name        string
+		err         bool
+		errText     string
+		emailParams EmailParams
+		smtpParams  SmtpParams
+	}{
+		{name: "empty"},
+		{name: "with template parse error",
+			err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				MsgTemplate: "{{",
+			}},
+		{name: "with verification template parse error",
+			err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From:                 "test@from",
+				VerificationTemplate: "{{",
+			},
+			smtpParams: SmtpParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+		{name: "normal creation",
+			err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From: "test@from",
+			},
+			smtpParams: SmtpParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+	}
+	for _, d := range testSet {
+		t.Run(d.name, func(t *testing.T) {
+			email, err := NewEmail(d.emailParams, d.smtpParams)
+
+			if d.err && d.errText == "" {
+				assert.Error(t, err)
+				assert.Nil(t, email)
+			} else if d.err && d.errText != "" {
+				assert.EqualError(t, err, d.errText)
+				assert.Nil(t, email)
+			} else {
+				assert.NoError(t, err)
+				assert.NotNil(t, email, "email returned")
+
+				assert.NotNil(t, email.msgTmpl, "e.template is set")
+				assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
+				assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
+				assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
+				if d.smtpParams.TimeOut == 0 {
+					assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
+				} else {
+					assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
+				}
+				assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
+				assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
+				assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
+				assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
+				assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
+			}
+		})
+	}
+}
+
+func TestEmailSendErrors(t *testing.T) {
+	var err error
+	e := Email{}
+	e.TokenGenFn = TokenGenFn
+
+	e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
+		"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
+	e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
+	assert.NoError(t, err)
+
+	e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"sending message to \"bad@example.org\" aborted due to canceled context")
+
+	e.smtp = &fakeTestSMTP{}
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
+		"error creating token for unsubscribe link: token generation error")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+}
+
+func TestEmailSend_ExitConditions(t *testing.T) {
+	email, err := NewEmail(EmailParams{}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "expecting email returned")
+	// prevent triggering e.autoFlush creation
+	emptyRequest := Request{Comment: store.Comment{ID: "999"}}
+	assert.NoError(t, email.Send(context.Background(), emptyRequest),
+		"Message without parent comment User.Email is not sent and returns nil")
+	requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
+	assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
+		"Message with parent comment User equals comment User is not sent and returns nil")
+}
+
+func TestEmailSendClientError(t *testing.T) {
+	var testSet = []struct {
+		name string
+		smtp *fakeTestSMTP
+		err  string
+	}{
+		{name: "failed to verify receiver", smtp: &fakeTestSMTP{fail: map[string]bool{"mail": true}},
+			err: "bad from address \"\": failed to verify sender"},
+		{name: "failed to verify sender", smtp: &fakeTestSMTP{fail: map[string]bool{"rcpt": true}},
+			err: "bad to address \"\": failed to verify receiver"},
+		{name: "failed to close connection", smtp: &fakeTestSMTP{fail: map[string]bool{"quit": true, "close": true}}},
+		{name: "failed to make email writer", smtp: &fakeTestSMTP{fail: map[string]bool{"data": true}},
+			err: "can't make email writer: failed to send"},
+	}
+	for _, d := range testSet {
+		t.Run(d.name, func(t *testing.T) {
+			e := Email{smtp: d.smtp}
+			if d.err != "" {
+				assert.EqualError(t, e.sendMessage(emailMessage{}), d.err,
+					"expected error for e.sendMessage")
+			} else {
+				assert.NoError(t, e.sendMessage(emailMessage{}),
+					"expected no error for e.sendMessage")
+			}
+		})
+	}
+	e := Email{}
+	e.smtp = nil
+	assert.Error(t, e.sendMessage(emailMessage{}),
+		"nil e.smtp should return error")
+	e.smtp = &fakeTestSMTP{}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage in normal flow")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"quit": true}}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage with failed smtpClient.Quit but successful smtpClient.Close")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"create": true}}
+	assert.EqualError(t, e.sendMessage(emailMessage{}), "failed to make smtp Create: failed to create client",
+		"e.send called without smtpClient set returns error")
+}
+
+func TestEmail_Send(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSmtp := fakeTestSMTP{}
+	email.smtp = &fakeSmtp
+	email.TokenGenFn = TokenGenFn
+	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
+	req := Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
+		Email:   "test@example.org",
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildMessageFromRequest(req)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: New reply to your comment for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
+Date: `)
+}
+
+func TestEmail_SendVerification(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SmtpParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSmtp := fakeTestSMTP{}
+	email.smtp = &fakeSmtp
+	email.TokenGenFn = TokenGenFn
+	req := Request{
+		Email: "test@example.org",
+		Verification: VerificationMetadata{
+			SiteID: "remark",
+			User:   "test_username",
+			Token:  "secret_",
+		},
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSmtp.readMail())
+	assert.Equal(t, 1, fakeSmtp.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSmtp.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `secret_`)
+	assert.NotContains(t, res, `https://example.org/`)
+	email.SubscribeURL = "https://example.org/subscribe.html?token="
+	res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `https://example.org/subscribe.html?token=3Dsecret_`)
+}
+
+func Test_emailClient_Create(t *testing.T) {
+	creator := emailClient{}
+	client, err := creator.Create(SmtpParams{})
+	assert.Error(t, err, "absence of address to connect results in error")
+	assert.Nil(t, client, "no client returned in case of error")
+}
+
+type fakeTestSMTP struct {
+	fail map[string]bool
+
+	buff       bytes.Buffer
+	mail, rcpt string
+	auth       bool
+	close      bool
+	quitCount  int
+	lock       sync.RWMutex
+}
+
+func (f *fakeTestSMTP) Create(SmtpParams) (smtpClient, error) {
+	if f.fail["create"] {
+		return nil, errors.New("failed to create client")
+	}
+	return f, nil
+}
+
+func (f *fakeTestSMTP) Auth(smtp.Auth) error { f.auth = true; return nil }
+
+func (f *fakeTestSMTP) Mail(m string) error {
+	f.lock.Lock()
+	f.mail = m
+	f.lock.Unlock()
+	if f.fail["mail"] {
+		return errors.New("failed to verify sender")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Rcpt(r string) error {
+	f.lock.Lock()
+	f.rcpt = r
+	f.lock.Unlock()
+	if f.fail["rcpt"] {
+		return errors.New("failed to verify receiver")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Quit() error {
+	f.lock.Lock()
+	f.quitCount++
+	f.lock.Unlock()
+	if f.fail["quit"] {
+		return errors.New("failed to quit")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Close() error {
+	f.close = true
+	if f.fail["close"] {
+		return errors.New("failed to close")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Data() (io.WriteCloser, error) {
+	if f.fail["data"] {
+		return nil, errors.New("failed to send")
+	}
+	return nopCloser{&f.buff}, nil
+}
+
+func (f *fakeTestSMTP) readRcpt() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.rcpt
+}
+
+func (f *fakeTestSMTP) readMail() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.mail
+}
+
+func (f *fakeTestSMTP) readQuitCount() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.quitCount
+}
+
+func TokenGenFn(user, _, _ string) (string, error) {
+	if user == "error" {
+		return "", errors.New("token generation error")
+	}
+	return "token", nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (nopCloser) Close() error {
+	return nil
+}

backend/app/notify/notify.go

@@ -16,7 +16,7 @@ import (
 type Service struct {
 	dataService  Store
 	destinations []Destination
-	queue        chan request
+	queue        chan Request
 
 	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
 	ctx    context.Context
@@ -26,17 +26,28 @@ type Service struct {
 // Destination defines interface for a given destination service, like telegram, email and so on
 type Destination interface {
 	fmt.Stringer
-	Send(ctx context.Context, req request) error
+	Send(ctx context.Context, req Request) error
 }
 
 // Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
-	Get(locator store.Locator, id string) (store.Comment, error)
+	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
 }
 
-type request struct {
-	comment store.Comment
-	parent  store.Comment
+// Request notification either about comment or about particular user verification
+type Request struct {
+	Comment      store.Comment        // if set sent notifications about new comment
+	parent       store.Comment        // fetched only in case Comment is set
+	Email        string               // if set (also) send email
+	Verification VerificationMetadata // if set sent verification notification
+}
+
+// VerificationMetadata required to send notify method verification message
+type VerificationMetadata struct {
+	SiteID string
+	User   string
+	Token  string
 }
 
 const defaultQueueSize = 100
@@ -50,7 +61,7 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	ctx, cancel := context.WithCancel(context.Background())
 	res := Service{
 		dataService:  dataService,
-		queue:        make(chan request, size),
+		queue:        make(chan Request, size),
 		destinations: destinations,
 		ctx:          ctx,
 		cancel:       cancel,
@@ -62,21 +73,25 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	return &res
 }
 
-// Submit comment to internal channel if not busy, drop if can't send
-func (s *Service) Submit(comment store.Comment) {
+// Submit Request to internal channel if not busy, drop if can't send
+func (s *Service) Submit(req Request) {
 	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
 		return
 	}
-	parentComment := store.Comment{}
-	if s.dataService != nil {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
-			parentComment = p
+	// parent comment is fetched only if comment is present in the Request
+	if s.dataService != nil && req.Comment.ParentID != "" {
+		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
+			req.parent = p
+			req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
+			if err != nil {
+				log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
+			}
 		}
 	}
 	select {
-	case s.queue <- request{comment: comment, parent: parentComment}:
+	case s.queue <- req:
 	default:
-		log.Printf("[WARN] can't send comment notification to queue, %+v", comment)
+		log.Printf("[WARN] can't send notification to queue, %+v", req.Comment)
 	}
 }
 

backend/app/notify/notify_mock.go

@@ -0,0 +1,40 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+type MockDest struct {
+	data   []Request
+	id     int
+	closed bool
+	lock   sync.Mutex
+}
+
+func (m *MockDest) Send(ctx context.Context, r Request) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.data = append(m.data, r)
+		log.Printf("sent %s -> %d", r.Comment.ID, m.id)
+	case <-ctx.Done():
+		log.Printf("ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+func (m *MockDest) Get() []Request {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]Request, len(m.data))
+	copy(res, m.data)
+	return res
+}
+func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }

backend/app/notify/notify_test.go

@@ -1,90 +1,89 @@
 package notify
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"math/rand"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestService_NoDestinations(t *testing.T) {
-	s := NewService(nil, 1)
+	s := NewService(nil, 0)
+	assert.Equal(t, defaultQueueSize, cap(s.queue))
 	assert.NotNil(t, s)
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "101"})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	assert.Equal(t, 3, len(d1.get()), "got all comments to d1")
-	assert.Equal(t, 3, len(d2.get()), "got all comments to d2")
+	require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
+	require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
 
-	assert.Equal(t, "100", d1.get()[0].comment.ID)
-	assert.Equal(t, "101", d1.get()[1].comment.ID)
-	assert.Equal(t, "102", d1.get()[2].comment.ID)
+	assert.Equal(t, "100", d1.Get()[0].Comment.ID)
+	assert.Equal(t, "101", d1.Get()[1].Comment.ID)
+	assert.Equal(t, "102", d1.Get()[2].Comment.ID)
 }
 
 func TestService_WithDrops(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
-	s.Submit(store.Comment{ID: "101"})
-	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
-	time.Sleep(time.Millisecond * 110)
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
+	time.Sleep(time.Millisecond * 11)
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
+	time.Sleep(time.Millisecond * 11)
 	s.Close()
 
-	s.Submit(store.Comment{ID: "111"}) // safe to send after close
+	s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
 
-	assert.Equal(t, 2, len(d1.get()), "one comment dropped from d1")
-	assert.Equal(t, 2, len(d2.get()), "one comment dropped from d2")
+	assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
+	assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
 }
 
 func TestService_Many(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 5, d1, d2)
 	assert.NotNil(t, s)
 
 	for i := 0; i < 10; i++ {
-		s.Submit(store.Comment{ID: fmt.Sprintf("%d", 100+i)})
-		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(200)))
+		s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
+		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
 	}
 	s.Close()
 	time.Sleep(time.Millisecond * 10)
 
-	assert.NotEqual(t, 10, len(d1.get()), "some comments dropped from d1")
-	assert.NotEqual(t, 10, len(d2.get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
 
 	assert.True(t, d1.closed)
 	assert.True(t, d2.closed)
 }
 
 func TestService_WithParent(t *testing.T) {
-	dest := &mockDest{id: 1}
+	dest := &MockDest{id: 1}
 	dataStore := &mockStore{data: map[string]store.Comment{}}
 
 	dataStore.data["p1"] = store.Comment{ID: "p1"}
@@ -93,63 +92,37 @@ func TestService_WithParent(t *testing.T) {
 	s := NewService(dataStore, 1, dest)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "c1", ParentID: "p1"})
+	s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "c11", ParentID: "p11"})
+	s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	destRes := dest.get()
-	assert.Equal(t, 2, len(destRes), "two comment notified")
-	assert.Equal(t, "p1", destRes[0].comment.ParentID)
+	destRes := dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ParentID)
 	assert.Equal(t, "p1", destRes[0].parent.ID)
-	assert.Equal(t, "p11", destRes[1].comment.ParentID)
+	assert.Equal(t, "p11", destRes[1].Comment.ParentID)
 	assert.Equal(t, "", destRes[1].parent.ID)
 }
 
 func TestService_Nop(t *testing.T) {
 	s := NopService
-	s.Submit(store.Comment{})
+	s.Submit(Request{Comment: store.Comment{}})
 	s.Close()
 	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
 }
 
-type mockDest struct {
-	data   []request
-	id     int
-	closed bool
-	lock   sync.Mutex
-}
-
-func (m *mockDest) Send(ctx context.Context, r request) error {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	select {
-	case <-time.After(100 * time.Millisecond):
-		m.data = append(m.data, r)
-		log.Printf("sent %s -> %d", r.comment.ID, m.id)
-	case <-ctx.Done():
-		log.Printf("ctx closed %d", m.id)
-		m.closed = true
-	}
-	return nil
-}
-
-func (m *mockDest) get() []request {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	res := make([]request, len(m.data))
-	copy(res, m.data)
-	return res
-}
-func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
-
 type mockStore struct{ data map[string]store.Comment }
 
-func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
 		return store.Comment{}, errors.New("no such id")
 	}
 	return res, nil
 }
+
+func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
+	return "", errors.New("no such user")
+}

backend/app/notify/telegram.go

@@ -40,7 +40,7 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	if res.timeout == 0 {
 		res.timeout = telegramTimeOut
 	}
-	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
+	log.Printf("[DEBUG] create new telegram notifier for chan %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -85,23 +85,27 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 }
 
 // Send to telegram channel
-func (t *Telegram) Send(ctx context.Context, req request) error {
+func (t *Telegram) Send(ctx context.Context, req Request) error {
+	if req.Comment.ID == "" {
+		// verification request received, send nothing
+		return nil
+	}
 	client := http.Client{Timeout: telegramTimeOut}
-	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
+	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
 
-	from := req.comment.User.Name
-	if req.comment.ParentID != "" {
+	from := req.Comment.User.Name
+	if req.Comment.ParentID != "" {
 		from += " → " + req.parent.User.Name
 	}
 	from = "*" + from + "*"
-	link := fmt.Sprintf("↦ [original comment](%s)", req.comment.Locator.URL+uiNav+req.comment.ID)
-	if req.comment.PostTitle != "" {
-		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
+	link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
+	if req.Comment.PostTitle != "" {
+		link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
 	}
 	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
 		t.apiPrefix, t.token, t.channelID)
 
-	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
+	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.Comment.Orig, link)
 	msg = html.UnescapeString(msg)
 	body := struct {
 		Text string `json:"text"`
@@ -113,10 +117,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/notify/telegram_test.go

@@ -30,7 +30,7 @@ func TestTelegram_New(t *testing.T) {
 	assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
 
 	_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "can't decode response:")
 
 	_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
@@ -58,24 +58,25 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NoError(t, err)
 	assert.NotNil(t, tb)
-	c := store.Comment{Text: "some text", ParentID: "1"}
+	c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
 	c.User.Name = "from"
 	cp := store.Comment{Text: "some parent text"}
 	cp.User.Name = "to"
 
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 	c.PostTitle = "test title"
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err, "should failed")
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
-	require.NotNil(t, err)
+	assert.Error(t, err, "should failed")
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
+	require.Error(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())
+	require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
 }
 
 func mockTelegramServer() *httptest.Server {

backend/app/rest/api/admin.go

@@ -9,41 +9,37 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService   *service.DataStore
-	cache         cache.LoadingCache
+	dataService   adminStore
+	cache         LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
+type adminStore interface {
+	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
+	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	IsBlocked(siteID string, userID string) bool
+	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
+	BlockedUsers(siteID string) ([]store.BlockedUser, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
+	SetVerified(siteID string, userID string, status bool) error
+	SetReadOnly(locator store.Locator, status bool) error
+	SetPin(locator store.Locator, commentID string, status bool) error
 }
 
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
@@ -70,7 +66,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+	if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
@@ -86,7 +82,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
 
-	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0, rest.GetUserOrEmpty(r))
 	if err != nil || len(ucomments) == 0 {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
@@ -95,7 +91,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, ucomments[0].User)
 }
 
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
+// GET /deleteme?token=jwt - delete all user comments and details by user's request. Gets info about deleted used from provided token
 // request made GET to allow direct click from the email sent by user
 func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -115,7 +111,13 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err := a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
@@ -150,6 +152,13 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
+
+	// delete comments for permanently blocked user.
+	if blockStatus && ttl == time.Duration(0) {
+		if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
+			log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
+		}
+	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
@@ -157,7 +166,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 // GET /blocked?site=siteID - list blocked users
 func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
+	users, err := a.dataService.BlockedUsers(siteID)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
@@ -237,42 +246,3 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

backend/app/rest/api/migrator.go

@@ -11,11 +11,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/migrator"
@@ -24,11 +23,12 @@ import (
 
 // Migrator rest with import and export controllers
 type Migrator struct {
-	Cache             cache.LoadingCache
+	Cache             LoadingCache
 	NativeImporter    migrator.Importer
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
 	NativeExporter    migrator.Exporter
+	UrlMapperMaker    migrator.MapperMaker
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -40,15 +40,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
@@ -108,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"status": "import request accepted"})
 }
 
-func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /wait?site=site-id
+// waits for migration operation (import or remap)
+func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	timeOut := time.Minute * 15
 	if v := r.URL.Query().Get("timeout"); v != "" {
@@ -162,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// POST /remap?site=site-id
+// remap urls in comments based on given rules (oldUrl newUrl)
+func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	// create new url-mapper from given rules in body
+	mapper, err := m.UrlMapperMaker(r.Body)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
+		return
+	}
+	defer r.Body.Close()
+
+	// start remap procedure with mapper
+	go func() {
+		m.setBusy(siteID, true)
+		defer m.setBusy(siteID, false)
+
+		// do export
+		fh, e := ioutil.TempFile("", "remark42_convert")
+		if e != nil {
+			log.Printf("[WARN] failed to make temp file %+v", e)
+			return
+		}
+		defer func() {
+			if e := os.Remove(fh.Name()); e != nil {
+				log.Printf("[WARN] failed to remove temp file %+v", e)
+			}
+		}()
+		log.Printf("[DEBUG] start export for site=%s", siteID)
+		if _, e := m.NativeExporter.Export(fh, siteID); e != nil {
+			log.Printf("[WARN] export failed with %+v", e)
+			return
+		}
+
+		if _, e = fh.Seek(0, 0); e != nil {
+			log.Printf("[WARN] failed to seek file %+v", e)
+			return
+		}
+
+		log.Printf("[DEBUG] start import for site=%s", siteID)
+		mappedReader := migrator.WithMapper(fh, mapper)
+		size, e := m.NativeImporter.Import(mappedReader, siteID)
+		if e != nil {
+			log.Printf("[WARN] import failed with %+v", e)
+			return
+		}
+
+		m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
+		log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size)
+	}()
+
+	render.Status(r, http.StatusAccepted)
+	render.JSON(w, r, R.JSON{"status": "convert request accepted"})
+}
+
 // runImport reads from tmpfile and import for given siteID and provider
 func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	m.setBusy(siteID, true)

backend/app/rest/api/migrator_test.go

@@ -3,60 +3,65 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/migrator"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
@@ -68,134 +73,156 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
-	assert.Nil(t, err)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=remark42&provider=native", contentType, bodyBuf)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
-	assert.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=wordpress", r)
+	assert.NoError(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native&secret=XYZ", r)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 150; i++ {
+	for i := 0; i < 50; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	client = &http.Client{Timeout: 5 * time.Second}
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 150; i++ {
+	for i := 0; i < 50; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	client = &http.Client{Timeout: 5 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=10ms", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
-	require.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
@@ -208,11 +235,11 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
 
@@ -222,57 +249,123 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42", nil)
+	require.NoError(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+func TestMigrator_Remap(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create 2 comments in https://remark42.com/demo/
+	c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
+	_, err := srv.DataService.Create(c1)
+	require.NoError(t, err)
+	c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
+	_, err = srv.DataService.Create(c2)
+	require.NoError(t, err)
+
+	// create 1 comment in https://remark42.com/demo-another/
+	c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
+	_, err = srv.DataService.Create(c3)
+	require.NoError(t, err)
+
+	// set url https://remark42.com/demo-another/ to be readonly
+	err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
+		URL:      "https://remark42.com/demo-another/",
+		ReadOnly: true,
+	}})
+	require.NoError(t, err)
+
+	// check that comments created as expected
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// we want remap urls to another domain - www.remark42.com
+	rules := "https://remark42.com/* https://www.remark42.com/*"
+	resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	waitForMigrationCompletion(t, ts)
+
+	// after remap finished we should find comments from new urls
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// should find nothing from previous url
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+}
+
+func TestMigrator_RemapReject(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// without admin credentials
+	client := &http.Client{Timeout: 1 * time.Second}
+	rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
+	require.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
 	resp, err := client.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
-}
-
-func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
-	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc := &Migrator{
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
-		Cache:             &cache.Nop{},
-		KeyStore:          adminStore,
-	}
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	teardown = func() {
-		ts.Close()
-		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
-	}
-	return ts, teardown
+	assert.Equal(t, "{\"site_id\":\"remark42\",\"status\":\"completed\"}\n", string(b))
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -5,9 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -19,9 +18,9 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/go-pkgz/rest/logger"
 	"github.com/pkg/errors"
 	"github.com/rakyll/statik/fs"
@@ -40,13 +39,15 @@ type Rest struct {
 
 	DataService      *service.DataStore
 	Authenticator    *auth.Service
-	Cache            cache.LoadingCache
+	Cache            LoadingCache
 	ImageProxy       *proxy.Image
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
 	ImageService     *image.Service
+	Streamer         *Streamer
 
+	AnonVote        bool
 	WebRoot         string
 	RemarkURL       string
 	ReadOnlyAge     int
@@ -55,14 +56,26 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter float64
+	UpdateLimiter      float64
+	EmailNotifications bool
+	EmojiEnabled       bool
+	SimpleView         bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	adminService admin
+	pubRest   public
+	privRest  private
+	adminRest admin
+	rssRest   rss
+}
+
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req lcw.FlusherRequest)                                        // evict matched records
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
@@ -159,24 +172,17 @@ func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 		Addr:              fmt.Sprintf(":%d", port),
 		Handler:           router,
 		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
-		IdleTimeout:       30 * time.Second,
+		// WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
+		IdleTimeout: 30 * time.Second,
 	}
 }
 
 func (s *Rest) routes() chi.Router {
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
 	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
 
-	s.adminService = admin{
-		dataService:   s.DataService,
-		migrator:      s.Migrator,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
+	s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
 
 	corsMiddleware := cors.New(cors.Options{
 		AllowedOrigins:   []string{"*"},
@@ -189,16 +195,18 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
+		r.Use(middleware.Timeout(5 * time.Second))
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
+		r.Use(middleware.Timeout(5 * time.Second))
 		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
@@ -209,6 +217,7 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
+			rava.Use(middleware.Timeout(5 * time.Second))
 			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
 			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
@@ -216,143 +225,227 @@ func (s *Rest) routes() chi.Router {
 
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(middleware.NoCache)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
-			ropen.Get("/info", s.infoCtrl)
-			ropen.Mount("/img", s.ImageProxy.Routes())
-			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Get("/find", s.pubRest.findCommentsCtrl)
+			ropen.Get("/id/{id}", s.pubRest.commentByIDCtrl)
+			ropen.Get("/comments", s.pubRest.findUserCommentsCtrl)
+			ropen.Get("/last/{limit}", s.pubRest.lastCommentsCtrl)
+			ropen.Get("/count", s.pubRest.countCtrl)
+			ropen.Post("/counts", s.pubRest.countMultiCtrl)
+			ropen.Get("/list", s.pubRest.listCtrl)
+			ropen.Post("/preview", s.pubRest.previewCommentCtrl)
+			ropen.Get("/info", s.pubRest.infoCtrl)
+			ropen.Get("/img", s.ImageProxy.Handler)
+
+			ropen.Route("/rss", func(rrss chi.Router) {
+				rrss.Get("/post", s.rssRest.postCommentsCtrl)
+				rrss.Get("/site", s.rssRest.siteCommentsCtrl)
+				rrss.Get("/reply", s.rssRest.repliesCtrl)
+			})
+
+		})
+
+		// open routes, streams, no send timeout
+		rapi.Route("/stream", func(rstream chi.Router) {
+			rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
+			rstream.Get("/info", s.pubRest.infoStreamCtrl)
+			rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
 		})
 
 		// open routes, cached
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
+			ropen.Get("/picture/{user}/{id}", s.pubRest.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(middleware.Timeout(30 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
-			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
+			rauth.Use(authMiddleware.Auth, matchSiteID, middleware.NoCache, logInfoWithBody)
+			rauth.Get("/user", s.privRest.userInfoCtrl)
+			rauth.Get("/userdata", s.privRest.userAllDataCtrl)
+		})
 
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		// admin routes, require auth and admin users only
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(middleware.Timeout(30 * time.Second))
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly, matchSiteID)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminRest.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminRest.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminRest.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminRest.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminRest.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminRest.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminRest.setPinCtrl)
+			radmin.Get("/blocked", s.adminRest.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminRest.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminRest.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminRest.migrator.exportCtrl)
+			radmin.Post("/import", s.adminRest.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
+			radmin.Post("/remap", s.adminRest.migrator.remapCtrl)
+			radmin.Get("/wait", s.adminRest.migrator.waitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(middleware.Timeout(10 * time.Second))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, matchSiteID)
 			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
-			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
+			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
+			rauth.Post("/comment", s.privRest.createCommentCtrl)
+			rauth.Put("/vote/{id}", s.privRest.voteCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.privRest.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Get("/email", s.privRest.getEmailCtrl)
+			rauth.With(rejectAnonUser).Post("/email/subscribe", s.privRest.sendEmailConfirmationCtrl)
+			rauth.With(rejectAnonUser).Post("/email/confirm", s.privRest.setConfirmedEmailCtrl)
+			rauth.With(rejectAnonUser).Delete("/email", s.privRest.deleteEmailCtrl)
 		})
 
+		// protected routes, anonymous rejected
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(middleware.Timeout(10 * time.Second))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, rejectAnonUser, matchSiteID)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+			rauth.Post("/picture", s.privRest.savePictureCtrl)
 		})
 
 	})
 
-	// respond to /robots.txt with the list of allowed paths
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
-				"/img", "/avatar", "/picture"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
+	// open routes on root level
+	router.Group(func(rroot chi.Router) {
+		rroot.Use(middleware.Timeout(10 * time.Second))
+		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
+		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
+		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
+		rroot.Get("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
+		rroot.Post("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
+	})
 
 	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
+	addFileServer(router, "/web", http.Dir(s.WebRoot), s.Version)
 	return router
 }
 
-func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+func (s *Rest) controllerGroups() (public, private, admin, rss) {
 
-	res = s.adminService.alterComments(comments, r) // apply admin's alteration
-
-	// prepare vote info for client view
-	vote := func(c store.Comment, r *http.Request) store.Comment {
-
-		c.Vote = 0 // default is "none" (not voted)
-
-		user, err := rest.GetUserInfo(r)
-		if err != nil {
-			c.Votes = nil // hide voters list and don't set Vote for non-authed user
-			return c
-		}
-
-		if v, ok := c.Votes[user.ID]; ok {
-			if v {
-				c.Vote = 1
-			} else {
-				c.Vote = -1
-			}
-		}
-
-		c.Votes = nil // hide voters list
-		return c
+	pubGrp := public{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		webRoot:          s.WebRoot,
+		streamer:         s.Streamer,
 	}
 
-	for i, c := range res {
-		c = vote(c, r)
-		res[i] = c
+	privGrp := private{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		authenticator:    s.Authenticator,
+		notifyService:    s.NotifyService,
+		remarkURL:        s.RemarkURL,
+		anonVote:         s.AnonVote,
 	}
 
-	return res
+	admGrp := admin{
+		dataService:   s.DataService,
+		migrator:      s.Migrator,
+		cache:         s.Cache,
+		authenticator: s.Authenticator,
+		readOnlyAge:   s.ReadOnlyAge,
+	}
+
+	rssGrp := rss{
+		dataService: s.DataService,
+		cache:       s.Cache,
+	}
+
+	return pubGrp, privGrp, admGrp, rssGrp
+}
+
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
+// GET /config?site=siteID - returns configuration
+func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	admins, _ := s.DataService.AdminStore.Admins(siteID)
+	emails, _ := s.DataService.AdminStore.Email(siteID)
+
+	cnf := struct {
+		Version            string   `json:"version"`
+		EditDuration       int      `json:"edit_duration"`
+		MaxCommentSize     int      `json:"max_comment_size"`
+		Admins             []string `json:"admins"`
+		AdminEmail         string   `json:"admin_email"`
+		Auth               []string `json:"auth_providers"`
+		AnonVote           bool     `json:"anon_vote"`
+		LowScore           int      `json:"low_score"`
+		CriticalScore      int      `json:"critical_score"`
+		PositiveScore      bool     `json:"positive_score"`
+		ReadOnlyAge        int      `json:"readonly_age"`
+		MaxImageSize       int      `json:"max_image_size"`
+		EmailNotifications bool     `json:"email_notifications"`
+		EmojiEnabled       bool     `json:"emoji_enabled"`
+		SimpleView         bool     `json:"simple_view"`
+	}{
+		Version:            s.Version,
+		EditDuration:       int(s.DataService.EditDuration.Seconds()),
+		MaxCommentSize:     s.DataService.MaxCommentSize,
+		Admins:             admins,
+		AdminEmail:         emails,
+		LowScore:           s.ScoreThresholds.Low,
+		CriticalScore:      s.ScoreThresholds.Critical,
+		PositiveScore:      s.DataService.PositiveScore,
+		ReadOnlyAge:        s.ReadOnlyAge,
+		MaxImageSize:       s.ImageService.Store.SizeLimit(),
+		EmailNotifications: s.EmailNotifications,
+		EmojiEnabled:       s.EmojiEnabled,
+		AnonVote:           s.AnonVote,
+		SimpleView:         s.SimpleView,
+	}
+
+	cnf.Auth = []string{}
+	for _, ap := range s.Authenticator.Providers() {
+		cnf.Auth = append(cnf.Auth, ap.Name())
+	}
+
+	if cnf.Admins == nil { // prevent json serialization to nil
+		cnf.Admins = []string{}
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, cnf)
 }
 
 // serves static files from /web or embedded by statik
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
+func addFileServer(r chi.Router, path string, root http.FileSystem, version string) {
 
 	var webFS http.Handler
 
@@ -374,15 +467,17 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	}
 	path += "*"
 
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// don't show dirs, just serve files
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-				http.NotFound(w, r)
-				return
-			}
-			webFS.ServeHTTP(w, r)
-		}))
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)),
+		middleware.Timeout(10*time.Second),
+		cacheControl(time.Hour, version),
+	).Get(path, func(w http.ResponseWriter, r *http.Request) {
+		// don't show dirs, just serve files
+		if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
+			http.NotFound(w, r)
+			return
+		}
+		webFS.ServeHTTP(w, r)
+	})
 }
 
 func encodeJSONWithHTML(v interface{}) ([]byte, error) {
@@ -448,3 +543,79 @@ func rejectAnonUser(next http.Handler) http.Handler {
 	}
 	return http.HandlerFunc(fn)
 }
+
+// matchSiteID is a middleware rejecting users with mismatch between site param and and User.SiteID
+func matchSiteID(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		// skip for basic auth user
+		if user.Name == "admin" && user.ID == "admin" {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		siteID := r.URL.Query().Get("site")
+		if siteID != "" && user.SiteID != siteID {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
+// cacheControl is a middleware setting cache expiration. Using url+version as etag
+func cacheControl(expiration time.Duration, version string) func(http.Handler) http.Handler {
+
+	etag := func(r *http.Request, version string) string {
+		s := version + ":" + r.URL.String()
+		return store.EncodeID(s)
+	}
+
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+			e := `"` + etag(r, version) + `"`
+			w.Header().Set("Etag", e)
+			w.Header().Set("Cache-Control", "max-age="+strconv.Itoa(int(expiration.Seconds())))
+
+			if match := r.Header.Get("If-None-Match"); match != "" {
+				if strings.Contains(match, e) {
+					w.WriteHeader(http.StatusNotModified)
+					return
+				}
+			}
+			h.ServeHTTP(w, r)
+		}
+		return http.HandlerFunc(fn)
+	}
+}
+
+func parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private.go

@@ -1,30 +1,80 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
+type private struct {
+	dataService      privStore
+	cache            LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	notifyService    *notify.Service
+	authenticator    *auth.Service
+	remarkURL        string
+	anonVote         bool
+}
+
+type privStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	EditComment(locator store.Locator, commentID string, req service.EditRequest) (comment store.Comment, err error)
+	Vote(req service.VoteReq) (comment store.Comment, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
+	SetUserEmail(siteID string, userID string, value string) (string, error)
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	ValidateComment(c *store.Comment) error
+	IsVerified(siteID string, userID string) bool
+	IsReadOnly(locator store.Locator) bool
+	IsBlocked(siteID string, userID string) bool
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+}
+
+const unsubscribeHtml = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Successfully unsubscribed</p>
+</div>
+</body>
+</html>
+`
+
 // POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
@@ -33,20 +83,26 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	user := rest.MustGetUserInfo(r)
+	if user.ID != "admin" && user.SiteID != comment.Locator.SiteID {
+		rest.SendErrorJSON(w, r, http.StatusForbidden,
+			fmt.Errorf("site mismatch, %q not allowed to post to %s", user.SiteID, comment.Locator.SiteID), "invalid site",
+			rest.ErrCommentValidation)
+		return
+	}
 
 	comment.PrepareUntrusted() // clean all fields user not supposed to set
 	comment.User = user
 	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
 
 	comment.Orig = comment.Text // original comment text, prior to md render
-	if err := s.DataService.ValidateComment(&comment); err != nil {
+	if err := s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 
 	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
+	if s.dataService.IsBlocked(comment.Locator.SiteID, comment.User.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
@@ -56,7 +112,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	id, err := s.DataService.Create(comment)
+	id, err := s.dataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
@@ -66,17 +122,17 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// DataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
+	// dataService modifies comment
+	finalComment, err := s.dataService.Get(comment.Locator, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+	s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
-	if s.NotifyService != nil {
-		s.NotifyService.Submit(finalComment)
+	if s.notifyService != nil {
+		s.notifyService.Submit(notify.Request{Comment: finalComment})
 	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -86,7 +142,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	edit := struct {
 		Text    string
@@ -107,7 +163,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	var currComment store.Comment
 	var err error
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
+	if currComment, err = s.dataService.Get(locator, id, rest.GetUserOrEmpty(r)); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
@@ -119,41 +175,53 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.CommentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
 	}
 
-	res, err := s.DataService.EditComment(locator, id, editReq)
+	res, err := s.dataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
 	if err != nil {
-		code := s.parseError(err, rest.ErrCommentRejected)
+		code := parseError(err, rest.ErrCommentRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
 // GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
+		user.Verified = s.dataService.IsVerified(siteID, user.ID)
+
+		email, err := s.dataService.GetUserEmail(siteID, user.ID)
+		if err != nil {
+			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+		}
+		if len(email) > 0 {
+			user.EmailSubscription = true
+		}
 	}
 
 	render.JSON(w, r, user)
 }
 
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
+	if !s.anonVote && strings.HasPrefix(user.ID, "anonymous_") {
+		http.Error(w, "Access denied", http.StatusForbidden)
+		return
+	}
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
 	log.Printf("[DEBUG] vote for comment %s", id)
@@ -166,23 +234,244 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// check if user blocked
-	if s.adminService.checkBlocked(locator.SiteID, user) {
+	if s.dataService.IsBlocked(locator.SiteID, user.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
+	req := service.VoteReq{
+		Locator:   locator,
+		CommentID: id,
+		UserID:    user.ID,
+		UserIP:    strings.Split(r.RemoteAddr, ":")[0],
+		Val:       vote,
+	}
+	comment, err := s.dataService.Vote(req)
 	if err != nil {
-		code := s.parseError(err, rest.ErrVoteRejected)
+		code := parseError(err, rest.ErrVoteRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
 	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
+// getEmailCtrl gets email address for authenticated user.
+// GET /email?site=siteID
+func (s *private) getEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	address, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// sendEmailConfirmationCtrl gets address and siteID from query, makes confirmation token and sends it to user.
+// GET /email/subscribe?site=siteID&address=someone@example.com
+func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	address := r.URL.Query().Get("address")
+	siteID := r.URL.Query().Get("site")
+	if address == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
+		return
+	}
+	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+	if address == existingAddress {
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
+		return
+	}
+	claims := token.Claims{
+		Handshake: &token.Handshake{ID: user.ID + "::" + address},
+		StandardClaims: jwt.StandardClaims{
+			Audience:  r.URL.Query().Get("site"),
+			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
+			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+			Issuer:    "remark42",
+		},
+	}
+
+	tkn, err := s.authenticator.TokenService().Token(claims)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to make verification token", rest.ErrInternal)
+		return
+	}
+
+	s.notifyService.Submit(
+		notify.Request{
+			Email: address,
+			Verification: notify.VerificationMetadata{
+				SiteID: siteID,
+				User:   user.Name,
+				Token:  tkn,
+			},
+		},
+	)
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// setConfirmedEmailCtrl uses provided token parameter (generated by sendEmailConfirmationCtrl) to set email and add it to user token
+// PUT /email/confirm?site=siteID&tkn=jwt
+func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 || elems[0] != user.ID {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	address := elems[1]
+
+	log.Printf("[DEBUG] set email for user %s", user.ID)
+
+	val, err := s.dataService.SetUserEmail(siteID, user.ID, address)
+	if err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set email for user", code)
+		return
+	}
+
+	// update User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	claims.User.Email = address
+	if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+		return
+	}
+	render.JSON(w, r, R.JSON{"updated": true, "address": val})
+}
+
+// POST/GET /email/unsubscribe.html?site=siteID&tkn=jwt - unsubscribe the user in token from email notifications
+func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	userID := elems[0]
+	address := elems[1]
+
+	existingAddress, err := s.dataService.GetUserEmail(siteID, userID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", userID, err)
+	}
+	if existingAddress == "" {
+		rest.SendErrorHTML(w, r, http.StatusConflict, errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
+		return
+	}
+	if address != existingAddress {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
+		return
+	}
+
+	log.Printf("[DEBUG] unsubscribe user %s", userID)
+
+	if err := s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token, if user has the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		log.Printf("[DEBUG] unsubscribed user doesn't have valid JWT token to update %s, %v", userID, err)
+	}
+	if claims.User != nil && claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err := tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHtml))
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, nil)
+	render.HTML(w, r, msg.String())
+}
+
+// DELETE /email?site=siteID - removes user's email
+func (s *private) deleteEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] remove email for user %s", user.ID)
+
+	if err := s.dataService.DeleteUserDetail(siteID, user.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	if claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+	render.JSON(w, r, R.JSON{"deleted": true})
+}
+
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
@@ -213,7 +502,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		comments, errUser := s.dataService.User(siteID, user.ID, 100, i*100, rest.GetUserOrEmpty(r))
 		if errUser != nil {
 			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
@@ -240,7 +529,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 // POST /deleteme?site_id=site - requesting delete of all user info
 // makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	siteID := r.URL.Query().Get("site")
 
@@ -260,18 +549,18 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
-	tokenStr, err := s.Authenticator.TokenService().Token(claims)
+	tokenStr, err := s.authenticator.TokenService().Token(claims)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
-	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
+	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.remarkURL, tokenStr)
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
 // POST /image - save image with form request
-func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 
 	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
@@ -286,7 +575,7 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = file.Close() }()
 
-	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	id, err := s.imageService.Save(header.Filename, user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return
@@ -295,37 +584,12 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"id": id})
 }
 
-func (s *Rest) isReadOnly(locator store.Locator) bool {
-	if s.ReadOnlyAge > 0 {
+func (s *private) isReadOnly(locator store.Locator) bool {
+	if s.readOnlyAge > 0 {
 		// check RO by age
-		if info, e := s.DataService.Info(locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+		if info, e := s.dataService.Info(locator, s.readOnlyAge); e == nil && info.ReadOnly {
 			return true
 		}
 	}
-	return s.DataService.IsReadOnly(locator) // ro manually
-}
-
-func (s *Rest) parseError(err error, defaultCode int) (code int) {
-	code = defaultCode
-
-	switch {
-	// voting errors
-	case strings.Contains(err.Error(), "can not vote for his own comment"):
-		code = rest.ErrVoteSelf
-	case strings.Contains(err.Error(), "already voted for"):
-		code = rest.ErrVoteDbl
-	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
-		code = rest.ErrVoteMax
-	case strings.Contains(err.Error(), "minimal score reached for comment"):
-		code = rest.ErrVoteMinScore
-
-	// edit errors
-	case strings.HasPrefix(err.Error(), "too late to edit"):
-		code = rest.ErrCommentEditExpired
-	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
-		code = rest.ErrCommentEditChanged
-
-	}
-
-	return code
+	return s.dataService.IsReadOnly(locator) // ro manually
 }

backend/app/rest/api/rest_public.go

@@ -4,51 +4,95 @@ import (
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
+type public struct {
+	dataService      pubStore
+	cache            LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	streamer         *Streamer
+	webRoot          string
+}
+
+type pubStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	UserCount(siteID, userID string) (int, error)
+	Count(locator store.Locator) (int, error)
+	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+
+	ValidateComment(c *store.Comment) error
+	IsReadOnly(locator store.Locator) bool
+	Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
+}
+
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
 	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
 		sort = "+" + sort[1:]
 	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
+
+	view := r.URL.Query().Get("view")
+	since, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
+		return
+	}
+	format := r.URL.Query().Get("format")
+	if format == "tree" {
+		since = time.Time{} // since doesn't make sense for tree
+	}
+
+	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v", locator, sort, format, since)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, sort)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
 		if e != nil {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.alterComments(comments, r)
+		comments = s.applyView(comments, view)
 		var b []byte
-		switch r.URL.Query().Get("format") {
+		switch format {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(comments, sort, s.readOnlyAge)
 			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*rest.Node{}
+				tree.Nodes = []*service.Node{}
 			}
-			if s.DataService.IsReadOnly(locator) {
+			if s.dataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
 			b, e = encodeJSONWithHTML(tree)
 		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
+			withInfo := commentsWithInfo{Comments: comments}
+			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
 				withInfo.Info = info
 			}
 			b, e = encodeJSONWithHTML(withInfo)
@@ -67,7 +111,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
@@ -81,23 +125,23 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	comment.User = user
 	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
+	if err = s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 	comment.Sanitize()
 	render.HTML(w, r, comment.Text)
 }
 
 // GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		info, e := s.dataService.Info(locator, s.readOnlyAge)
 		if e != nil {
 			return nil, e
 		}
@@ -114,8 +158,53 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
+func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+	log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
+	fn := func() steamEventFn {
+		lastTS := sinceTs
+		lastCount := 0
+
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				info, e := s.dataService.Info(locator, s.readOnlyAge)
+				if e != nil {
+					return nil, e
+				}
+				// cache update used as indication of post update. comparing lastTS for no-cache.
+				// removal won't update lastTS, count check will catch it.
+				if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
+					upd = true
+				}
+				lastTS = info.LastTS
+				lastCount = info.Count
+				return encodeJSONWithHTML(info)
+			})
+			if err != nil {
+				return "info", data, false, err
+			}
+
+			return "info", data, upd, nil
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
+// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
+// limited with "since" param
+func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get last comments for %s", siteID)
 
@@ -124,13 +213,18 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		limit = 0
 	}
 
+	sinceTime, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Last(siteID, limit, sinceTime, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
@@ -146,8 +240,47 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
+func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] get last comments stream for %s", siteID)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+	if sinceTs.IsZero() {
+		sinceTs = time.Now()
+	}
+
+	fn := func() steamEventFn {
+		sinceTime := sinceTs
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
+				if e != nil {
+					return nil, e
+				}
+				if len(comments) > 0 {
+					sinceTime = comments[0].Timestamp
+					upd = true
+				}
+				sinceTime = time.Now()
+				return encodeJSONWithHTML(comments)
+			})
+			return "last", data, upd, err
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	id := chi.URLParam(r, "id")
 	siteID := r.URL.Query().Get("site")
@@ -155,12 +288,11 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
 
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
+	comment, err := s.dataService.Get(store.Locator{SiteID: siteID, URL: url}, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -169,7 +301,7 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
@@ -187,14 +319,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
 	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.User(siteID, userID, limit, 0, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
+		count, e := s.dataService.UserCount(siteID, userID)
 		if e != nil {
 			return nil, e
 		}
@@ -212,53 +343,10 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		PositiveScore  bool     `json:"positive_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-		MaxImageSize   int      `json:"max_image_size"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.DataService.AdminStore.Admins(siteID),
-		AdminEmail:     s.DataService.AdminStore.Email(siteID),
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		PositiveScore:  s.DataService.PositiveScore,
-		ReadOnlyAge:    s.ReadOnlyAge,
-		MaxImageSize:   s.ImageService.Store.SizeLimit(),
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers() {
-		cnf.Auth = append(cnf.Auth, ap.Name())
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
 // GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
+	count, err := s.dataService.Count(locator)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
@@ -267,7 +355,7 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /counts?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
@@ -277,12 +365,12 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	h := sha1.Sum([]byte(k)) //nolint
+	h := sha1.Sum([]byte(k)) // nolint
 	sha := base64.URLEncoding.EncodeToString(h[:])
 
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		counts, e := s.dataService.Counts(siteID, posts)
 		if e != nil {
 			return nil, e
 		}
@@ -300,7 +388,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 	siteID := r.URL.Query().Get("site")
 	limit, skip := 0, 0
@@ -313,8 +401,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		posts, e := s.dataService.List(siteID, limit, skip)
 		if e != nil {
 			return nil, e
 		}
@@ -327,12 +415,12 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
-		log.Printf("[WARN] can't render posts lits for site %s", siteID)
+		log.Printf("[WARN] can't render posts list for site %s", siteID)
 	}
 }
 
 // GET /picture/{user}/{id} - get picture
-func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 
 	imgContentType := func(img string) string {
 		img = strings.ToLower(img)
@@ -348,7 +436,7 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
-	imgRdr, size, err := s.ImageService.Load(id)
+	imgRdr, size, err := s.imageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
 		return
@@ -364,7 +452,11 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer imgRdr.Close()
+	defer func() {
+		if e := imgRdr.Close(); e != nil {
+			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
+		}
+	}()
 
 	w.Header().Set("Content-Type", imgContentType(id))
 	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
@@ -373,3 +465,50 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.webRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}
+
+func (s *public) applyView(comments []store.Comment, view string) []store.Comment {
+	if strings.EqualFold(view, "user") {
+		projection := make([]store.Comment, len(comments))
+		for i, c := range comments {
+			p := store.Comment{
+				ID:   c.ID,
+				User: c.User,
+			}
+			projection[i] = p
+		}
+		return projection
+	}
+	return comments
+}
+
+func (s *public) parseSince(r *http.Request) (time.Time, error) {
+	sinceTs := time.Time{}
+	if since := r.URL.Query().Get("since"); since != "" {
+		unixTS, e := strconv.ParseInt(since, 10, 64)
+		if e != nil {
+			return time.Time{}, errors.Wrap(e, "can't translate since parameter")
+		}
+		sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+	}
+	return sinceTs, nil
+}

backend/app/rest/api/rest_public_test.go

@@ -1,21 +1,24 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -32,11 +35,15 @@ func TestRest_Preview(t *testing.T) {
 	defer teardown()
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
+
+	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_PreviewWithMD(t *testing.T) {
@@ -56,13 +63,12 @@ BKT
 	text = strings.Replace(text, "BKT", "```", -1)
 	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
 	j = strings.Replace(j, "\n", "\\n", -1)
-	t.Log(j)
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
 }
 
@@ -70,53 +76,55 @@ func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id1 := addComment(t, c1, ts)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id2 := addComment(t, c2, ts)
 
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "<p>test test #1</p>\n", comments.Comments[0].Text)
+	assert.Equal(t, "<p>test test #2</p>\n", comments.Comments[1].Text)
 	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
 	assert.Equal(t, 2, comments.Info.Count)
 	assert.Equal(t, false, comments.Info.ReadOnly)
 	assert.True(t, comments.Info.FirstTS.Before(comments.Info.LastTS))
 
 	// get sorted by -time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=-time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=-time")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[1].ID)
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	tree := service.Tree{}
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(tree.Nodes))
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(tree.Nodes))
 	assert.Equal(t, 1, len(tree.Nodes[0].Replies))
 	assert.Equal(t, 2, tree.Info.Count)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
@@ -128,28 +136,28 @@ func TestRest_FindAge(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is fresh")
 
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is old")
 }
@@ -159,90 +167,148 @@ func TestRest_FindReadOnly(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
 
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -2),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	// set post to read-only
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	tree := rest.Tree{}
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	tree := service.Tree{}
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	tree = service.Tree{}
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is writable")
 }
 
+func TestRest_FindUserView(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&view=user")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	id1 := addComment(t, c1, ts)
+
+	c2 := store.Comment{Text: "test test #2", ParentID: id1,
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	id2 := addComment(t, c2, ts)
+
+	assert.NotEqual(t, id1, id2)
+
+	// get sorted by +time with view=user
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time&view=user")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, id1, comments.Comments[0].ID)
+	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[1].Text)
+}
+
 func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "[]\n", res, "empty last should return empty list")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
+	ts1 := time.Now().UnixNano() / 1000000
 	addComment(t, c1, ts)
 	id1 := addComment(t, c1, ts)
+	time.Sleep(10 * time.Millisecond)
+	ts2 := time.Now().UnixNano() / 1000000
 	id2 := addComment(t, c2, ts)
 
-	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "should have 2 comments")
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts1))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.Equal(t, id1, comments[1].ID)
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts2))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(comments), "should have 1 comments")
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	res, code = get(t, ts.URL+"/api/v1/last/X?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/X?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
-	assert.Nil(t, err)
+	err = srv.DataService.Delete(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
+	assert.NoError(t, err)
 	srv.Cache.Flush(cache.FlusherRequest{})
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
-	t.Logf("%+v", comments)
+
+	_, code = get(t, ts.URL+"/api/v1/last/2?site=remark42-BLAH")
+	assert.Equal(t, 500, code)
 }
 
 func TestRest_FindUserComments(t *testing.T) {
@@ -250,9 +316,9 @@ func TestRest_FindUserComments(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
 	addComment(t, c1, ts)
@@ -264,10 +330,10 @@ func TestRest_FindUserComments(t *testing.T) {
 	err := srv.DataService.Delete(c2.Locator, id, store.SoftDelete)
 	assert.NoError(t, err)
 
-	_, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=blah")
+	_, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=blah")
 	assert.Equal(t, 400, code, "noting for user blah")
 
-	res, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=dev")
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
 	assert.Equal(t, 200, code)
 
 	resp := struct {
@@ -276,21 +342,26 @@ func TestRest_FindUserComments(t *testing.T) {
 	}{}
 
 	err = json.Unmarshal([]byte(res), &resp)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(resp.Comments), "should have 3 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(resp.Comments), "should have 3 comments")
 	assert.Equal(t, 4, resp.Count, "should have 3 count")
+
+	// user comment sorted with -time
+	assert.True(t, resp.Comments[0].Timestamp.After(resp.Comments[1].Timestamp))
+	assert.True(t, resp.Comments[1].Timestamp.After(resp.Comments[2].Timestamp))
 }
 
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=radio-t")
+	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=remark42")
 	assert.Equal(t, 200, code)
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, user)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 
 func TestRest_Count(t *testing.T) {
@@ -298,9 +369,9 @@ func TestRest_Count(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -308,18 +379,21 @@ func TestRest_Count(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3.0, j["count"])
 
-	body, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah2")
+	body, code = get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah2")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2.0, j["count"])
+
+	_, code = get(t, ts.URL+"/api/v1/count?site=remark42-BLAH&url=https://radio-t.com/blah1XXX")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_Counts(t *testing.T) {
@@ -327,9 +401,9 @@ func TestRest_Counts(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -337,18 +411,22 @@ func TestRest_Counts(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	j := []store.PostInfo{}
 	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
 		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_List(t *testing.T) {
@@ -356,9 +434,9 @@ func TestRest_List(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -366,15 +444,18 @@ func TestRest_List(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
 	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
 	assert.Equal(t, 3, pi[1].Count)
+
+	_, code = get(t, ts.URL+"/api/v1/list?site=remark42-BLAH")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_ListWithSkipAndLimit(t *testing.T) {
@@ -382,11 +463,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah3"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -396,11 +477,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	addComment(t, c3, ts)
 	addComment(t, c3, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42&skip=1&limit=2")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	require.Equal(t, 2, len(pi))
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
@@ -412,11 +493,11 @@ func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/config?site=remark42")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 300., j["edit_duration"])
 	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
 	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
@@ -426,46 +507,185 @@ func TestRest_Config(t *testing.T) {
 	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
 	assert.Equal(t, 10000., j["max_image_size"])
-	t.Logf("%+v", j)
+	assert.Equal(t, true, j["emoji_enabled"].(bool))
 }
 
 func TestRest_Info(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 
 	user := store.User{ID: "user1", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
+	require.NoError(t, err, "%+v", err)
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	body, code := get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 
 	info := store.PostInfo{}
 	err = json.Unmarshal([]byte(body), &info)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
 		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 	assert.Equal(t, exp, info)
 
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42-no&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
 }
 
+func TestRest_InfoStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 800 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	<-done
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
+	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
+
+	_, code = get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah123")
+	assert.Equal(t, 500, code)
+}
+
+func TestRest_InfoStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+}
+
+func TestRest_InfoStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_InfoStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 1500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 5; i++ {
+			time.Sleep(300 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1", nil)
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	defer r.Body.Close()
+	<-ctx.Done()
+	<-done
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	recs := strings.Count(string(body), "data:")
+	require.Equal(t, 1, recs, "should have 1 event:\n", string(body))
+	assert.Contains(t, string(body), `"count":2`)
+}
+
+func TestRest_InfoStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 900 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 10; i++ {
+			time.Sleep(15 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1&since=12345678")
+	assert.Equal(t, 200, code)
+	<-done
+	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
+	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
+}
+
 func TestRest_Robots(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -476,3 +696,168 @@ func TestRest_Robots(t *testing.T) {
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
 		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }
+
+func TestRest_LastCommentsStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
+	require.NoError(t, err)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	defer r.Body.Close()
+	<-done
+	body, err := ioutil.ReadAll(r.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 9*3, len(recs), "9 events")
+	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
+}
+
+func TestRest_LastCommentsStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_LastCommentsStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	<-done
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	assert.True(t, len(recs) < 30, "less 10 events")
+}
+
+func TestRest_LastCommentsStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+	assert.Equal(t, 200, code, "all streams closed, good to go again")
+}
+
+func TestRest_LastCommentsStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			time.Sleep(50 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42&since=123456", nil)
+	require.NoError(t, err)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	<-done
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "should be 10 events, including first record:\n", recs)
+}
+
+func postComment(t *testing.T, url string) {
+	resp, err := post(t, url+"/api/v1/comment",
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+}

backend/app/rest/api/rest_test.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -19,12 +21,14 @@ import (
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
@@ -34,28 +38,36 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testHTML = "/tmp/test-remark.html"
-var getStartedHTML = "/tmp/getstarted.html"
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
 
-var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
 
-var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
+
+var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/web/test-remark.html")
+	testHtmlName := "test-remark.html"
+	testHTMLFile := os.TempDir() + "/" + testHtmlName
+	err := ioutil.WriteFile(testHTMLFile, []byte("some html"), 0700)
+	assert.NoError(t, err)
+
+	body, code := get(t, ts.URL+"/web/"+testHtmlName)
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "some html", body)
+	_ = os.Remove(testHTMLFile)
 }
 
 func TestRest_GetStarted(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
+	getStartedHTML := os.TempDir() + "/getstarted.html"
 	err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	body, code := get(t, ts.URL+"/index.html")
 	assert.Equal(t, 200, code)
@@ -69,15 +81,20 @@ func TestRest_GetStarted(t *testing.T) {
 
 func TestRest_Shutdown(t *testing.T) {
 	srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
+	done := make(chan bool)
 
+	// without waiting for channel close at the end goroutine will stay alive after test finish
+	// which would create data race with next test
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(200 * time.Millisecond)
 		srv.Shutdown()
+		close(done)
 	}()
 
 	st := time.Now()
 	srv.Run(0)
 	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
+	<-done
 }
 
 func TestRest_filterComments(t *testing.T) {
@@ -96,6 +113,7 @@ func TestRest_filterComments(t *testing.T) {
 }
 
 func TestRest_RunStaticSSLMode(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: auth.NewService(auth.Opts{
 			AvatarStore:       avatar.NewLocalFS("/tmp"),
@@ -105,18 +123,19 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		ImageProxy: &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Static,
-			Port:    8443,
+			Port:    sslPort,
 			Key:     "../../cmd/testdata/key.pem",
 			Cert:    "../../cmd/testdata/cert.pem",
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
-		srv.Run(38080)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -130,40 +149,42 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38080/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
-	resp, err = client.Get("https://localhost:8443/ping")
-	require.Nil(t, err)
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	srv.Shutdown()
 }
 
 func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: &auth.Service{},
 		ImageProxy:    &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Auto,
-			Port:    8443,
+			Port:    sslPort,
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
 		// can't check https server locally, just only http server
-		srv.Run(38081)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -172,11 +193,11 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38081/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	srv.Shutdown()
 }
@@ -249,28 +270,93 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 }
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}
+
+func TestRest_cacheControl(t *testing.T) {
+
+	tbl := []struct {
+		url     string
+		version string
+		exp     time.Duration
+		etag    string
+		maxAge  int
+	}{
+		{"http://example.com/foo", "v1", time.Hour, "b433be1ea19edaee9dc92ca4b895b6bdf3c058cb", 3600},
+		{"http://example.com/foo2", "v1", 10 * time.Hour, "6d8466aef3246c1057452561acddf7ad9d0d99e0", 36000},
+		{"http://example.com/foo", "v2", time.Hour, "481700c52aab0dfbca99f3ffc2a4fbb27884c114", 3600},
+		{"https://example.com/foo", "v2", time.Hour, "bebd4f1b87f474792c4e75e5affe31fbf67f5778", 3600},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := httptest.NewRequest("GET", tt.url, nil)
+			w := httptest.NewRecorder()
+
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h.ServeHTTP(w, req)
+			resp := w.Result()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			t.Logf("%+v", resp.Header)
+			assert.Equal(t, `"`+tt.etag+`"`, resp.Header.Get("Etag"))
+			assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds())), resp.Header.Get("Cache-Control"))
+
+		})
+	}
+
+}
+
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 
-	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
-	os.Remove(testDb)
-	os.Remove(testHTML)
-	os.RemoveAll("/tmp/ava-remark42")
-	os.RemoveAll("/tmp/pics-remark42")
+	tmp := os.TempDir()
+	var testDb string
+	// pick a file name which is not in use for sure
+	for i := 0; i < 10; i++ {
+		testDb = fmt.Sprintf("/%s/test-remark-%d.db", tmp, rand.Int31())
+		_, err := os.Stat(testDb)
+		if err != nil {
+			break
+		}
+	}
+	_ = os.RemoveAll(tmp + "/ava-remark42")
+	_ = os.RemoveAll(tmp + "/pics-remark42")
 
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
+	require.NoError(t, err)
 
-	memCache, err := cache.NewMemoryCache()
-	assert.NoError(t, err)
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
 
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
-		Interface:              b,
+		Engine:                 b,
 		EditDuration:           5 * time.Minute,
 		MaxCommentSize:         4000,
-		AdminStore:             adminStore,
+		AdminStore:             astore,
 		MaxVotes:               service.UnlimitedVotes,
 		RestrictedWordsMatcher: restrictedWordsMatcher,
 	}
@@ -280,17 +366,17 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
 			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
-			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
+			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
-		WebRoot:   "/tmp",
+		WebRoot:   tmp,
 		RemarkURL: "https://demo.remark42.com",
 		ImageService: &image.Service{
 			Store: &image.FileSystem{
-				Location:   "/tmp/pics-remark42",
+				Location:   tmp + "/pics-remark42",
 				Partitions: 100,
 				MaxSize:    10000,
-				Staging:    "/tmp/pics-remark42/staging",
+				Staging:    tmp + "/pics-remark42/staging",
 			},
 			TTL: time.Millisecond * 100,
 		},
@@ -302,30 +388,34 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
-			Cache:             &cache.Nop{},
-			KeyStore:          adminStore,
+			UrlMapperMaker:    migrator.NewUrlMapper,
+			Cache:             memCache,
+			KeyStore:          astore,
 		},
+		Streamer: &Streamer{
+			Refresh:   100 * time.Millisecond,
+			TimeOut:   5 * time.Second,
+			MaxActive: 100,
+		},
+		NotifyService: notify.NopService,
+		EmojiEnabled:  true,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-
 	ts = httptest.NewServer(srv.routes())
 
 	teardown = func() {
 		ts.Close()
 		require.NoError(t, srv.DataService.Close())
-		os.Remove(testDb)
-		os.Remove(testHTML)
-		os.RemoveAll("/tmp/ava-remark42")
-		os.RemoveAll("/tmp/pics-remark42")
+		_ = os.Remove(testDb)
+		_ = os.RemoveAll(tmp + "/ava-remark42")
+		_ = os.RemoveAll(tmp + "/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
-// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+// fake auth middleware make user authenticated and uses query's fake_id for ID and fake_name for Name
 func fakeAuth(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Query().Get("fake_id") != "" {
@@ -341,14 +431,14 @@ func fakeAuth(next http.Handler) http.Handler {
 
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	return string(body), r.StatusCode
 }
 
-func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
 	client := http.Client{Timeout: 5 * time.Second}
 	if token != "" {
 		r.Header.Set("X-JWT", token)
@@ -359,53 +449,86 @@ func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	b, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(b), r.StatusCode
 }
 
 func getWithAdminAuth(t *testing.T, url string) (string, int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(body), r.StatusCode
 }
 func post(t *testing.T, url string, body string) (*http.Response, error) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	return client.Do(req)
 }
 
 func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	b, err := json.Marshal(c)
-	require.Nil(t, err, "can't marshal comment %+v", c)
+	require.NoError(t, err, "can't marshal comment %+v", c)
 
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	crResp := R.JSON{}
 	err = json.Unmarshal(b, &crResp)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}

backend/app/rest/api/rss.go

@@ -5,9 +5,8 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
 	"github.com/pkg/errors"
 
@@ -15,38 +14,40 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
+type rss struct {
+	dataService rssStore
+	cache       LoadingCache
+}
+
+type rssStore interface {
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error)
+}
+
 const maxRssItems = 20
-const maxLastCommentsReply = 5000
 const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[DEBUG] get rss for post %+v", locator)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Find(locator, "-time", rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
+		feed, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -63,23 +64,22 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) siteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss for site %s", siteID)
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Last(siteID, maxRssItems, time.Time{}, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
+		feed, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -95,43 +95,24 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/reply?user=userID&site=siteID
-func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) repliesCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
-	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() (res []byte, e error) {
-		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+	data, err := s.cache.Get(key, func() (res []byte, e error) {
+
+		replies, userName, e := s.dataService.UserReplies(siteID, userID, maxRssItems, maxReplyDuration)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.alterComments(comments, r)
-		replies := []store.Comment{}
-		for _, c := range comments {
-			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
-				break
-			}
-			if c.User.ID != userID {
-				userName = c.User.Name
-			}
-			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
-				var pc store.Comment
-				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
-					return nil, errors.Wrap(e, "can't get parent comment")
-				}
-				if pc.User.ID == userID {
-					replies = append(replies, c)
-				}
-			}
-		}
 
-		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
+		feed, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -146,7 +127,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
+func (s *rss) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
 	if description == "" {
 		description = "comment updates"
@@ -175,9 +156,10 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment, description strin
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
+			parentComment, err := s.dataService.Get(c.Locator, c.ParentID, store.User{})
 			if err == nil {
 				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
+				f.Description = f.Description + "<blockquote><p>" + parentComment.Snippet(300) + "</p></blockquote>"
 			} else {
 				log.Printf("[WARN] failed to get info about parent comment, %s", err)
 			}

backend/app/rest/api/rss_test.go

@@ -22,7 +22,7 @@ func TestServer_RssPost(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "1234567890",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	id1, err := rst.DataService.Create(c1)
@@ -30,7 +30,7 @@ func TestServer_RssPost(t *testing.T) {
 	assert.Equal(t, "1234567890", id1)
 	pubDate := time.Now().Format(time.RFC1123Z)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -54,7 +54,7 @@ func TestServer_RssPost(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
+	_, code = get(t, ts.URL+"/api/v1/rss/post?site=remark42-bad&url=https://radio-t.com/blah1")
 	assert.Equal(t, 400, code)
 }
 
@@ -69,13 +69,13 @@ func TestServer_RssSite(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:      "comment-id-2",
 		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -85,15 +85,15 @@ func TestServer_RssSite(t *testing.T) {
 	require.NoError(t, err)
 
 	require.NoError(t, err)
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/site?site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 		  <channel>
 		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
-		    <description>site comment for radio-t</description>
+		    <link>remark42</link>
+		    <description>site comment for remark42</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -132,14 +132,14 @@ func TestServer_RssWithReply(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-id-2",
 		ParentID: "comment-id-1",
 		Text:     "xyz test",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:     store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -148,7 +148,7 @@ func TestServer_RssWithReply(t *testing.T) {
 	_, err = rst.DataService.Create(c2)
 	require.NoError(t, err)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah10")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -161,7 +161,7 @@ func TestServer_RssWithReply(t *testing.T) {
 		    <item>
 		      <title>developer one &gt; developer one</title>
 		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
-		      <description>xyz test</description>
+              <description>xyz test&lt;blockquote&gt;&lt;p&gt;test 123&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>developer one</author>
               <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
@@ -192,34 +192,34 @@ func TestServer_RssReplies(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-1",
 		Text:    "c1",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "user1", Name: "user1"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-2",
 		Text:     "reply to c1 from user2",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user2", Name: "user2"},
 	}
 	c3 := store.Comment{
 		ID:       "comment-3",
 		Text:     "reply to c1 from user3",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user3", Name: "user3"},
 	}
 	c4 := store.Comment{
 		ID:       "comment-4",
 		Text:     "reply to c2 from developer one",
 		ParentID: "comment-2",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "dev", Name: "developer one"},
 	}
 	c5 := store.Comment{
 		ID:      "comment-5",
 		Text:    "developer one",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "dev", Name: "developer one"},
 	}
 
@@ -235,19 +235,19 @@ func TestServer_RssReplies(t *testing.T) {
 	require.NoError(t, err)
 
 	// replies to c1 (user1). Must be [c3, c2]
-	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 	    <channel>
 	        <title>Remark42 comments</title>
-	        <link>radio-t</link>
-	        <description>replies to user2</description>
+	        <link>remark42</link>
+	        <description>replies to user1</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
-		      <description>reply to c1 from user3</description>
+		      <description>reply to c1 from user3&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user3</author>
               <guid>comment-3</guid> 
 		      <pubDate>%s</pubDate>
@@ -255,7 +255,7 @@ func TestServer_RssReplies(t *testing.T) {
 			<item>
 		      <title>user2 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
-		      <description>reply to c1 from user2</description>
+		      <description>reply to c1 from user2&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user2</author>
               <guid>comment-2</guid> 
 		      <pubDate>%s</pubDate>
@@ -265,7 +265,7 @@ func TestServer_RssReplies(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t-bad")
+	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42-bad")
 	assert.Equal(t, 400, code)
 }
 

backend/app/rest/api/ssl.go

@@ -86,7 +86,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager {
 // makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
 func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	cfg := makeTLSConfig()
+	cfg := s.makeTLSConfig()
 	cfg.GetCertificate = m.GetCertificate
 	server.TLSConfig = cfg
 	return server
@@ -95,7 +95,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocer
 // makeHTTPSServer makes https server for static mode
 func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	server.TLSConfig = makeTLSConfig()
+	server.TLSConfig = s.makeTLSConfig()
 	return server
 }
 
@@ -109,7 +109,7 @@ func (s *Rest) getRemarkHost() string {
 	return u.Hostname()
 }
 
-func makeTLSConfig() *tls.Config {
+func (s *Rest) makeTLSConfig() *tls.Config {
 	return &tls.Config{
 		PreferServerCipherSuites: true,
 		CipherSuites: []uint16{
@@ -119,7 +119,7 @@ func makeTLSConfig() *tls.Config {
 			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			// tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
 		},
 		MinVersion: tls.VersionTLS12,
 		CurvePreferences: []tls.CurveID{

backend/app/rest/api/ssl_test.go

@@ -33,7 +33,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
@@ -62,28 +62,28 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
 
 	// check acme http challenge
 	req, err := http.NewRequest("GET", ts.URL+"/.well-known/acme-challenge/token123", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Host = "localhost" // for passing hostPolicy check
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 404, resp.StatusCode)
 
 	err = m.Cache.Put(context.Background(), "token123+http-01", []byte("token"))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "token", string(body))
 }

backend/app/rest/api/stream.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// Streamer creates endless stream of \n separated json records send to remote client
+type Streamer struct {
+	TimeOut     time.Duration
+	Refresh     time.Duration
+	MaxActive   int32
+	activeCount int32
+}
+
+type steamEventFn func() (event string, data []byte, upd bool, err error)
+
+type steamEventResp struct {
+	data  []byte
+	event string
+	err   error
+}
+
+// Activate starts blocking function streaming update created by eventFn to ResponseWriter
+// canceled on context or inactivity timeout
+// note: eventFn is a closure needed to allow state management inside eventFn
+func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
+	updCh := s.eventsCh(ctx, eventFn())
+
+	count := atomic.AddInt32(&s.activeCount, 1)
+	defer atomic.AddInt32(&s.activeCount, -1)
+	if count > s.MaxActive {
+		return errors.New("too many streams")
+	}
+
+	if ww, ok := w.(http.ResponseWriter); ok {
+		ww.Header().Set("Content-Type", "text/event-stream")
+		ww.Header().Set("Connection", "keep-alive")
+		ww.Header().Set("Cache-Control", "no-cache")
+	}
+
+	for {
+		select {
+		case <-ctx.Done(): // request closed by remote client
+			log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
+			return nil
+		case <-time.After(s.TimeOut): // request closed by timeout
+			log.Printf("[DEBUG] stream closed due to timeout")
+			return nil
+		case resp, ok := <-updCh: // new update
+			if !ok { // closed updCh
+				return nil
+			}
+			if resp.err != nil {
+				return resp.err
+			}
+
+			// make server-sent event record
+			// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+			if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
+				return errors.Wrap(e, "send to stream failed")
+			}
+			if fw, okFlush := w.(http.Flusher); okFlush {
+				fw.Flush()
+			}
+		}
+	}
+}
+
+// populate updates to chan, break on context close
+func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
+	ch := make(chan steamEventResp)
+	go func() {
+		tick := time.NewTicker(s.Refresh)
+		defer func() {
+			close(ch)
+			tick.Stop()
+		}()
+		for {
+			select {
+			case <-ctx.Done(): // request closed by remote client
+				return
+			case <-tick.C:
+				event, resp, upd, err := fn()
+				if err != nil {
+					ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
+					return
+				}
+				if upd {
+					ch <- steamEventResp{event: event, data: resp, err: nil}
+				}
+			}
+		}
+	}()
+	return ch
+}

backend/app/rest/api/stream_test.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStream_Timeout(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 || n > 10 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	err := s.Activate(context.Background(), eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}
+
+func TestStream_Cancel(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err := s.Activate(ctx, eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}

backend/app/rest/httperrors.go

@@ -1,7 +1,10 @@
 package rest
 
 import (
+	"bytes"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"net/url"
 	"runtime"
@@ -35,9 +38,51 @@ const (
 	ErrAssetNotFound      = 18 // requested file not found
 )
 
+const errorHtml = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">{{.Error}}: {{.Details}}.</p>
+</div>
+</body>
+</html>
+`
+
+// errTmplData store data for error message
+type errTmplData struct {
+	Error   string
+	Details string
+}
+
+// SendErrorHTML makes html body with provided template and responds with provided http status code,
+// error code is not included in render as it is intended for UI developers and not for the users
+func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err := tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("error").Parse(errorHtml))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	render.Status(r, httpStatusCode)
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, errTmplData{
+		Error:   err.Error(),
+		Details: details,
+	})
+	render.HTML(w, r, msg.String())
+}
+
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
 func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
 	render.Status(r, httpStatusCode)
 	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
@@ -56,14 +101,10 @@ func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details strin
 	if pc, file, line, ok := runtime.Caller(2); ok {
 		fnameElems := strings.Split(file, "/")
 		funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-		srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
+		srcFileInfo = fmt.Sprintf("[%s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
 			line, funcNameElems[len(funcNameElems)-1])
 	}
 
-	remoteIP := r.RemoteAddr
-	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
-		remoteIP = remoteIP[:pos]
-	}
-	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
-		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s",
+		details, err, httpStatusCode, errCode, uinfoStr, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -26,23 +27,51 @@ func TestSendErrorJSON(t *testing.T) {
 	defer ts.Close()
 
 	resp, err := http.Get(ts.URL + "/error")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
 	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
+func TestSendErrorHTML(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/error" {
+			t.Log("http err request", r.URL)
+			SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL + "/error")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 500, resp.StatusCode)
+
+	assert.NotContains(t, string(body), `987`, "user html should not contain internal error code")
+	assert.Contains(t, string(body), `error details 123456`)
+	assert.Contains(t, string(body), `error 500`)
+}
+
 func TestErrorDetailsMsg(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		req.RemoteAddr = "1.2.3.4"
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (123) - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsg]")
 	}
 	callerFn()
 }
@@ -50,11 +79,14 @@ func TestErrorDetailsMsg(t *testing.T) {
 func TestErrorDetailsMsgWithUser(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
+		require.NoError(t, err)
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
-		require.Nil(t, err)
+		require.NoError(t, err)
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (34567) - test/id - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsgWithUser]")
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,119 +1,61 @@
 package proxy
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // Image extracts image src from comment's html and provides proxy for them
 // this is needed to keep remark42 running behind of HTTPS serve all images via https
 type Image struct {
-	RemarkURL string
-	RoutePath string
-	Enabled   bool
-	Timeout   time.Duration
+	RemarkURL     string
+	RoutePath     string
+	HTTP2HTTPS    bool
+	CacheExternal bool
+	Timeout       time.Duration
+	ImageService  *image.Service
 }
 
-// Convert all img src links without https to proxied links
+// Convert img src links to proxied links depends on enabled options
 func (p Image) Convert(commentHTML string) string {
-	if !p.Enabled || strings.HasPrefix(p.RemarkURL, "http://") {
-		return commentHTML
+	if p.CacheExternal {
+		imgs, err := p.extract(commentHTML, func(img string) bool { return !strings.HasPrefix(img, p.RemarkURL) })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
 	}
 
-	imgs, err := p.extract(commentHTML)
-	if err != nil {
-		return commentHTML
+	if p.HTTP2HTTPS && !strings.HasPrefix(p.RemarkURL, "http://") {
+		imgs, err := p.extract(commentHTML, func(img string) bool { return strings.HasPrefix(img, "http://") })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
 	}
 
-	return p.replace(commentHTML, imgs)
+	return commentHTML
 }
 
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
-	if !p.Enabled {
-		return router
-	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-			return
-		}
-
-		timeout := 60 * time.Second // default
-		if p.Timeout > 0 {
-			timeout = p.Timeout
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		defer cancel()
-
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-			var e error
-			req, e := http.NewRequest("GET", string(src), nil)
-			if e != nil {
-				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-			}
-			resp, e = client.Do(req.WithContext(ctx))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
-
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
-
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
-		}
-	})
-	return router
-}
-
-// extract gets all non-https images and return list of src
-func (p Image) extract(commentHTML string) ([]string, error) {
+// extract gets all images matching predicate and return list of src
+func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]string, error) {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return nil, errors.Wrap(err, "can't create document")
@@ -121,7 +63,7 @@ func (p Image) extract(commentHTML string) ([]string, error) {
 	result := []string{}
 	doc.Find("img").Each(func(i int, s *goquery.Selection) {
 		if im, ok := s.Attr("src"); ok {
-			if strings.HasPrefix(im, "http://") {
+			if imgSrcPred(im) {
 				result = append(result, im)
 			}
 		}
@@ -131,7 +73,6 @@ func (p Image) extract(commentHTML string) ([]string, error) {
 
 // replace img links in commentHTML with route to proxy, base64 encoded original link
 func (p Image) replace(commentHTML string, imgs []string) string {
-
 	for _, img := range imgs {
 		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
 		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
@@ -140,3 +81,142 @@ func (p Image) replace(commentHTML string, imgs []string) string {
 
 	return commentHTML
 }
+
+// Handler returns http handler respond to proxied request
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
+	if !p.HTTP2HTTPS && !p.CacheExternal {
+		// TODO: we might need to find a better way to handle it. If admin enables caching/proxy and disables it later on
+		// all comments that got converted will lose their images. We can't just return a redirect (it will open an ability
+		// to redirect anywhere). We can probably continue proxying these images (but need to make sure this behavior is
+		// documented) or, better, provide a way to migrate back converted comments.
+		http.Error(w, "none of the proxy features are enabled", http.StatusNotImplemented)
+		return
+	}
+
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
+		return
+	}
+
+	imgURL := string(src)
+	var imgReader io.ReadCloser
+	imgID, err := cachedImgID(imgURL)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse image url "+imgURL, rest.ErrAssetNotFound)
+		return
+	}
+	if p.CacheExternal {
+		imgReader, _, err = p.ImageService.Load(imgID)
+		if err != nil {
+			imgReader = nil
+		}
+	}
+	if imgReader == nil {
+		imgReader, err = p.downloadImage(context.Background(), imgURL)
+		if err != nil {
+			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
+			return
+		}
+		if p.CacheExternal {
+			var buf bytes.Buffer
+			// We need to duplicate data into a new buffer because `cacheImage` would read provider Reader
+			// and we would need another one to read data for response
+			p.cacheImage(io.TeeReader(imgReader, &buf), imgID)
+			if err := imgReader.Close(); err != nil {
+				log.Printf("[WARN] can't close image reader, %s", err)
+			}
+			imgReader = ioutil.NopCloser(&buf)
+		}
+	}
+	defer func() {
+		if e := imgReader.Close(); e != nil {
+			log.Printf("[WARN] can't close image reader, %s", e)
+		}
+	}()
+
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	w.Header().Add("Content-Type", "image/*")
+	_, err = io.Copy(w, imgReader)
+	if err != nil {
+		log.Printf("[WARN] can't copy image stream, %s", err)
+	}
+}
+
+// cache image from provided Reader using given ID
+func (p Image) cacheImage(r io.Reader, imgID string) {
+	id, err := p.ImageService.SaveWithID(imgID, r)
+	if err != nil {
+		log.Printf("[WARN] unable to save image to the storage: %+v", err)
+	}
+	// In the future we can do something smarter than just committing everything (eg, some kind of LFU/LRU)
+	if err := p.ImageService.Commit(id); err != nil {
+		log.Printf("[WARN] unable to commit image %s", imgID)
+	}
+}
+
+// download an image. Returns a Reader which has to be closed by a caller
+func (p Image) downloadImage(ctx context.Context, imgURL string) (io.ReadCloser, error) {
+	log.Printf("[DEBUG] downloading image %s", imgURL)
+
+	timeout := 60 * time.Second // default
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	client := http.Client{Timeout: 30 * time.Second}
+	var resp *http.Response
+	err := repeater.NewDefault(5, time.Second).Do(ctx, func() error {
+		var e error
+		req, e := http.NewRequest("GET", imgURL, nil)
+		if e != nil {
+			return errors.Wrapf(e, "failed to make request for %s", imgURL)
+		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
+	})
+	if err != nil {
+		log.Print(err.Error())
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
+	}
+
+	imgData, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("unable to read image body")
+	}
+	return ioutil.NopCloser(bytes.NewBuffer(imgData)), nil
+}
+
+func sha1Str(s string) string {
+	return fmt.Sprintf("%x", sha1.Sum([]byte(s))) // nolint
+}
+
+// generates ID for a cached image.
+// ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
+// <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
+// <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
+//                              (as accurate as deduplication based on potentially mutable url can be)
+func cachedImgID(imgURL string) (string, error) {
+	parsedURL, err := url.Parse(imgURL)
+	if err != nil {
+		return "", errors.Wrapf(err, "can parse url %s", imgURL)
+	}
+	return fmt.Sprintf("cached_images/%s-%s", sha1Str(parsedURL.Hostname()), sha1Str(imgURL)), nil
+}

backend/app/rest/proxy/image_test.go

@@ -1,17 +1,22 @@
 package proxy
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 func TestPicture_Extract(t *testing.T) {
@@ -45,87 +50,168 @@ func TestPicture_Extract(t *testing.T) {
 			[]string{},
 		},
 	}
-	img := Image{Enabled: true}
+	img := Image{HTTP2HTTPS: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			res, err := img.extract(tt.inp, func(src string) bool { return strings.HasPrefix(src, "http://") })
+			assert.NoError(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
 func TestPicture_Replace(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
 		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 }
 
 func TestImage_Routes(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
+	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	t.Logf("%+v", resp.Header)
 	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 404, resp.StatusCode)
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
-func TestImage_RoutesTimedOut(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
-	router := img.Routes()
+func TestImage_Routes_CachingImage(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  &image.Service{Store: &imageStore},
+	}
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
+
+	imgURL := httpSrv.URL + "/image/img1.png"
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, int64(0), nil)
+	imageStore.On("SaveWithID", mock.Anything, mock.Anything).Once().Run(func(args mock.Arguments) { _, _ = ioutil.ReadAll(args.Get(1).(io.Reader)) }).Return("", nil)
+	imageStore.On("Commit", mock.Anything).Once().Return(nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "123", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+	imageStore.AssertCalled(t, "SaveWithID", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+sha1Str(imgURL), mock.Anything)
+	imageStore.AssertCalled(t, "Commit", mock.Anything)
+}
+
+func TestImage_Routes_Using_Cachded_Image(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  &image.Service{Store: &imageStore},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	// In order to validate that cached data is used cache "will return" some other data from what http server would
+	imageReader := ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("%256s", "X"))))
+	imageStore.On("Load", mock.Anything).Once().Return(imageReader, int64(256), nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "256", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/*", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+}
+
+func TestImage_RoutesTimedOut(t *testing.T) {
+	img := Image{HTTP2HTTPS: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	require.NoError(t, err)
+	assert.Equal(t, 404, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
 }
 
-func TestPicture_Convert(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+func TestPicture_Convert_ProxyMode(t *testing.T) {
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
 	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://radio-t.com/img3.png"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
-	img = Image{Enabled: true, RoutePath: "/img", RemarkURL: "http://example.com"}
+	img = Image{HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "http://example.com"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "http:// remark url, no proxy")
 
-	img = Image{Enabled: false, RoutePath: "/img"}
+	img = Image{HTTP2HTTPS: false, RoutePath: "/img"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
 }
 
+func TestPicture_Convert_CachingMode(t *testing.T) {
+	img := Image{CacheExternal: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+
+	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="https://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cHM6Ly9yYWRpby10LmNvbS9pbWczLnBuZw=="/> xyz <img src="https://remark42.com/img?src=aHR0cHM6Ly9pbWFnZXMucGV4ZWxzLmNvbS82NzYzNi9pbWc0LmpwZWc=">`, r)
+
+	r = img.Convert(`<img src="https://remark42.com/pictures/1.png"/>`)
+	assert.Equal(t, `<img src="https://remark42.com/pictures/1.png"/>`, r)
+
+	img = Image{CacheExternal: false, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/>`)
+	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/>`, r)
+
+	// both Caching and Proxy are enabled
+	img = Image{CacheExternal: true, HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+}
+
 func imgHTTPServer(t *testing.T) *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/image/img1.png" {

backend/app/rest/user.go

@@ -35,17 +35,27 @@ func GetUserInfo(r *http.Request) (user store.User, err error) {
 		Admin:    u.IsAdmin(),
 		Verified: u.BoolAttr("verified"),
 		Blocked:  u.BoolAttr("blocked"),
+		SiteID:   u.Audience,
 	}, nil
+}
 
+// GetUserOrEmpty attempts to get user info from request and returns empty object if failed
+func GetUserOrEmpty(r *http.Request) store.User {
+	user, err := GetUserInfo(r)
+	if err != nil {
+		return store.User{}
+	}
+	return user
 }
 
 // SetUserInfo sets user into request context
 func SetUserInfo(r *http.Request, user store.User) *http.Request {
 	u := token.User{
-		ID:      user.ID,
-		Name:    user.Name,
-		Picture: user.Picture,
-		IP:      user.IP,
+		ID:       user.ID,
+		Name:     user.Name,
+		Picture:  user.Picture,
+		IP:       user.IP,
+		Audience: user.SiteID,
 		Attributes: map[string]interface{}{
 			"blocked":  user.Blocked,
 			"verified": user.Verified,

backend/app/rest/user_test.go

@@ -10,14 +10,14 @@ import (
 
 func TestUser_GetUserInfo(t *testing.T) {
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
+	assert.Error(t, err, "no user info")
 
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
+	r = SetUserInfo(r, store.User{Name: "test", ID: "id", SiteID: "test"})
 	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "test", ID: "id", SiteID: "test"}, u)
 }
 
 func TestUSer_MustGetUserInfo(t *testing.T) {
@@ -28,12 +28,12 @@ func TestUSer_MustGetUserInfo(t *testing.T) {
 	}()
 
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_ = MustGetUserInfo(r)
 	assert.Fail(t, "should panic")
 
 	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
 	u := MustGetUserInfo(r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
 }

backend/app/store/admin/admin.go

@@ -3,6 +3,7 @@ package admin
 
 import (
 	"errors"
+	"strings"
 
 	log "github.com/go-pkgz/lgr"
 )
@@ -10,29 +11,35 @@ import (
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key() (key string, err error)
-	Admins(siteID string) (ids []string)
-	Email(siteID string) (email string)
+	Admins(siteID string) (ids []string, err error)
+	Email(siteID string) (email string, err error)
+	Enabled(siteID string) (ok bool, err error)
+	OnEvent(siteID string, et EventType) error
 }
 
-// StaticStore implements keys.Store with a single, predefined key
+// EventType indicates type of the event
+type EventType int
+
+// enum of all event types
+const (
+	EvCreate EventType = iota
+	EvDelete
+	EvUpdate
+	EvVote
+)
+
+// StaticStore implements keys.Store with a single set of admins and email for all sites
 type StaticStore struct {
 	admins []string
 	email  string
 	key    string
-}
-
-// Key returns static key for all sites, allows empty site
-func (s *StaticStore) Key() (key string, err error) {
-	if s.key == "" {
-		return "", errors.New("empty key for static key store")
-	}
-	return s.key, nil
+	sites  []string
 }
 
 // NewStaticStore makes StaticStore instance with given key
-func NewStaticStore(key string, admins []string, email string) *StaticStore {
+func NewStaticStore(key string, sites []string, admins []string, email string) *StaticStore {
 	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
-	return &StaticStore{key: key, admins: admins, email: email}
+	return &StaticStore{key: key, sites: sites, admins: admins, email: email}
 }
 
 // NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
@@ -40,12 +47,36 @@ func NewStaticKeyStore(key string) *StaticStore {
 	return &StaticStore{key: key, admins: []string{}, email: ""}
 }
 
+// Key returns static key, same for all sites
+func (s *StaticStore) Key() (key string, err error) {
+	if s.key == "" {
+		return "", errors.New("empty key for static key store")
+	}
+	return s.key, nil
+}
+
 // Admins returns static list of admin's ids, the same for all sites
-func (s *StaticStore) Admins(string) (ids []string) {
-	return s.admins
+func (s *StaticStore) Admins(string) (ids []string, err error) {
+	return s.admins, nil
 }
 
 // Email gets static email address
-func (s *StaticStore) Email(string) (email string) {
-	return s.email
+func (s *StaticStore) Email(string) (email string, err error) {
+	return s.email, nil
 }
+
+// Enabled if always true for StaticStore
+func (s *StaticStore) Enabled(site string) (ok bool, err error) {
+	if len(s.sites) == 0 {
+		return true, nil
+	}
+	for _, allowedSite := range s.sites {
+		if strings.EqualFold(allowedSite, site) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// OnEvent doesn nothing for StaticStore
+func (s *StaticStore) OnEvent(siteID string, et EventType) error { return nil }

backend/app/store/admin/admin_test.go

@@ -3,61 +3,29 @@ package admin
 import (
 	"testing"
 
-	"github.com/globalsign/mgo"
-	"github.com/go-pkgz/mongo"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestStaticStore_Get(t *testing.T) {
-	var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com")
+	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"}, []string{"123", "xyz"}, "aa@example.com")
 
 	k, err := ks.Key()
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 
-	a := ks.Admins("any")
+	a, err := ks.Admins("s1")
+	assert.NoError(t, err)
 	assert.Equal(t, []string{"123", "xyz"}, a)
 
-	email := ks.Email("blah")
+	email, err := ks.Email("s2")
+	assert.NoError(t, err)
 	assert.Equal(t, "aa@example.com", email)
-}
 
-func TestMongoStore_Get(t *testing.T) {
-	conn, err := mongo.MakeTestConnection(t)
-	require.NoError(t, err)
-	var ms Store = NewMongoStore(conn, "secret")
-
-	recs := []mongoRec{
-		{"site1", []string{"i11", "i12"}, "e1"},
-		{"site2", []string{"i21", "i22"}, "e2"},
-	}
-	err = conn.WithCollection(func(coll *mgo.Collection) error {
-		if e1 := coll.Insert(recs[0]); e1 != nil {
-			return e1
-		}
-		return coll.Insert(recs[1])
-	})
-	require.NoError(t, err)
-
-	admins := ms.Admins("site1")
-	assert.Equal(t, []string{"i11", "i12"}, admins)
-	email := ms.Email("site1")
-	assert.Equal(t, "e1", email)
-	key, err := ms.Key()
+	enabled, err := ks.Enabled("s3")
 	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
+	assert.Equal(t, true, enabled)
 
-	admins = ms.Admins("site2")
-	assert.Equal(t, []string{"i21", "i22"}, admins)
-	email = ms.Email("site2")
-	assert.Equal(t, "e2", email)
-	key, err = ms.Key()
+	enabled, err = ks.Enabled("serr")
 	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("no-site-in-db")
-	assert.Equal(t, []string{}, admins)
-	email = ms.Email("no-site-in-db")
-	assert.Equal(t, "", email)
+	assert.Equal(t, false, enabled)
 }

backend/app/store/admin/mongo.go

@@ -1,56 +0,0 @@
-package admin
-
-import (
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	log "github.com/go-pkgz/lgr"
-
-	"github.com/go-pkgz/mongo"
-)
-
-// MongoStore implements admin.Store with mongo backend
-type MongoStore struct {
-	connection *mongo.Connection
-	key        string
-}
-
-type mongoRec struct {
-	SiteID string   `bson:"site"`
-	IDs    []string `bson:"admin_ids"`
-	Email  string   `bson:"admin_email"`
-}
-
-// NewMongoStore makes admin Store for mongo's connection
-func NewMongoStore(conn *mongo.Connection, key string) *MongoStore {
-	log.Printf("[DEBUG] make mongo admin store with %+v", conn)
-	return &MongoStore{connection: conn, key: key}
-}
-
-// Key executes find by siteID and returns substructure with secret key
-func (m *MongoStore) Key() (key string, err error) {
-	return m.key, nil
-}
-
-// Admins executes find by siteID and returns admins ids
-func (m *MongoStore) Admins(siteID string) (ids []string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return []string{}
-	}
-	return resp.IDs
-}
-
-// Email executes find by siteID and returns admin's email
-func (m *MongoStore) Email(siteID string) (email string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return ""
-	}
-	return resp.Email
-}

backend/app/store/admin/remote.go

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	jrpc.Client
+}
+
+// Key returns the key, same for all sites
+func (r *RPC) Key() (key string, err error) {
+	resp, err := r.Call("admin.key")
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &key)
+	return key, err
+}
+
+// Admins returns list of admin's ids for given site
+func (r *RPC) Admins(siteID string) (ids []string, err error) {
+	resp, err := r.Call("admin.admins", siteID)
+	if err != nil {
+		return []string{}, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+		return []string{}, err
+	}
+	return ids, nil
+}
+
+// Email gets email address for given site
+func (r *RPC) Email(siteID string) (email string, err error) {
+	resp, err := r.Call("admin.email", siteID)
+	if err != nil {
+		return "", err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &email); err != nil {
+		return "", err
+	}
+	return email, nil
+}
+
+// Enabled returns true if allowed
+func (r *RPC) Enabled(siteID string) (ok bool, err error) {
+	resp, err := r.Call("admin.enabled", siteID)
+	if err != nil {
+		return false, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ok); err != nil {
+		return false, err
+	}
+	return ok, nil
+}
+
+// OnEvent reacts (register) events about data modification
+func (r *RPC) OnEvent(siteID string, et EventType) error {
+	_, err := r.Call("admin.event", siteID, et)
+	if err != nil {
+		return err
+	}
+	return nil
+}

backend/app/store/admin/remote_test.go

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemote_Key(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.key","id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Admins(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.admins","params":"site-1","id":1}`,
+		`{"result":["id1","id2"],"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Admins("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Email(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.email","params":"site-1","id":1}`,
+		`{"result":"bbb@example.com","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Email("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, "bbb@example.com", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Enables(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.enabled","params":"site-1","id":1}`,
+		`{"result":true,"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Enabled("site-1")
+	assert.NoError(t, err)
+	assert.True(t, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_OnEvent(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.event","params":["site-1",2],"id":1}`, `{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.OnEvent("site-1", EvUpdate)
+	assert.NoError(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/store/comment.go

@@ -3,6 +3,7 @@ package store
 import (
 	"html/template"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/microcosm-cc/bluemonday"
@@ -10,21 +11,22 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID          string          `json:"id" bson:"_id"`
-	ParentID    string          `json:"pid"`
-	Text        string          `json:"text"`
-	Orig        string          `json:"orig,omitempty"`
-	User        User            `json:"user"`
-	Locator     Locator         `json:"locator"`
-	Score       int             `json:"score"`
-	Votes       map[string]bool `json:"votes,omitempty"`
-	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
-	Controversy float64         `json:"controversy,omitempty"`
-	Timestamp   time.Time       `json:"time" bson:"time"`
-	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
-	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
-	PostTitle   string          `json:"title,omitempty" bson:"title"`
+	ID          string                 `json:"id" bson:"_id"`
+	ParentID    string                 `json:"pid"`
+	Text        string                 `json:"text"`
+	Orig        string                 `json:"orig,omitempty"`
+	User        User                   `json:"user"`
+	Locator     Locator                `json:"locator"`
+	Score       int                    `json:"score"`
+	Votes       map[string]bool        `json:"votes,omitempty"`
+	VotedIPs    map[string]VotedIPInfo `json:"voted_ips,omitempty"` // voted ips (hashes) with TS
+	Vote        int                    `json:"vote"`                // vote for the current user, -1/1/0.
+	Controversy float64                `json:"controversy,omitempty"`
+	Timestamp   time.Time              `json:"time" bson:"time"`
+	Edit        *Edit                  `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Pin         bool                   `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool                   `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string                 `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post
@@ -55,6 +57,12 @@ type BlockedUser struct {
 	Until time.Time `json:"time"`
 }
 
+// VotedIPInfo keeps timestamp and voting value (direction). Used as VotedIPs value
+type VotedIPInfo struct {
+	Timestamp time.Time
+	Value     bool
+}
+
 // DeleteMode defines how much comment info will be erased
 type DeleteMode int
 
@@ -66,6 +74,7 @@ const (
 
 // Maximum length for URL text shortening.
 const shortURLLen = 48
+const snippetLen = 200
 
 // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
 // autogen fields and reset everything users not supposed to provide
@@ -104,6 +113,35 @@ func (c *Comment) Sanitize() {
 	c.Text = p.Sanitize(c.Text)
 	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = template.HTMLEscapeString(c.User.Name)
+	c.User.Name = c.escapeHtmlWithSome(c.User.Name)
 	c.User.Picture = p.Sanitize(c.User.Picture)
 }
+
+// Snippet from comment's text
+func (c *Comment) Snippet(limit int) string {
+	if limit <= 0 {
+		limit = snippetLen
+	}
+	cleanText := strings.Replace(c.Text, "\n", " ", -1)
+	size := len([]rune(cleanText))
+	if size < limit {
+		return cleanText
+	}
+	snippet := []rune(cleanText)[:size]
+	// go back in snippet and found the first space
+	for i := len(snippet) - 1; i >= 0; i-- {
+		if snippet[i] == ' ' {
+			snippet = snippet[:i]
+			break
+		}
+	}
+	return string(snippet) + " ..."
+}
+
+func (c *Comment) escapeHtmlWithSome(inp string) string {
+	res := template.HTMLEscapeString(inp)
+	res = strings.Replace(res, "&#34;", "\"", -1)
+	res = strings.Replace(res, "&#39;", "'", -1)
+	res = strings.Replace(res, "&amp;", "&", -1)
+	return res
+}

backend/app/store/comment_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -27,13 +28,25 @@ func TestComment_Sanitize(t *testing.T) {
 		{
 			inp: Comment{
 				Text: "blah 123" + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 			out: Comment{
 				Text: `blah 123` + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123", User: User{Name: "name <> & ' ` \""}},
+			out: Comment{Text: `blah &amp; &amp; 123`, User: User{Name: "name &lt;&gt; & ' ` \""}},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -120,3 +133,25 @@ func TestComment_SetDeletedHard(t *testing.T) {
 	assert.False(t, comment.Pin)
 	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
 }
+
+func TestComment_Snippet(t *testing.T) {
+	tbl := []struct {
+		limit int
+		inp   string
+		out   string
+	}{
+		{0, "", ""},
+		{-1, "test\nblah", "test blah"},
+		{5, "test\nblah", "test ..."},
+		{5, "xyz12345 xxx", "xyz12345 ..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			c := Comment{Text: tt.inp}
+			out := c.Snippet(tt.limit)
+			assert.Equal(t, tt.out, out)
+		})
+	}
+}

backend/app/store/engine/bolt_accessor.go

@@ -1,540 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
-	dbs map[string]*bolt.DB
-}
-
-const (
-	// top level buckets
-	postsBucketName    = "posts"
-	lastBucketName     = "last"
-	userBucketName     = "users"
-	blocksBucketName   = "block"
-	infoBucketName     = "info"
-	readonlyBucketName = "readonly"
-	verifiedBucketName = "verified"
-
-	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-)
-
-// BoltSite defines single site param
-type BoltSite struct {
-	FileName string // full path to boltdb
-	SiteID   string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
-	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
-	result := BoltDB{dbs: make(map[string]*bolt.DB)}
-	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
-		}
-
-		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
-			infoBucketName, readonlyBucketName, verifiedBucketName}
-		err = db.Update(func(tx *bolt.Tx) error {
-			for _, bktName := range topBuckets {
-				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
-				}
-			}
-			return nil
-		})
-
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create top level bucket)")
-		}
-
-		result.dbs[site.SiteID] = db
-		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
-	}
-	return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
-	bdb, err := b.db(comment.Locator.SiteID)
-	if err != nil {
-		return "", err
-	}
-
-	if b.IsReadOnly(comment.Locator) {
-		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
-	}
-
-	err = bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
-		if e != nil {
-			return e
-		}
-
-		// check if key already in store, reject doubles
-		if postBkt.Get([]byte(comment.ID)) != nil {
-			return errors.Errorf("key %s already in store", comment.ID)
-		}
-
-		// serialize comment to json []byte for bolt and save
-		if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
-			return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
-		}
-
-		ref := b.makeRef(comment)
-
-		// add reference to comment to "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		e = lastBkt.Put(commentTs, ref)
-		if e != nil {
-			return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
-		}
-
-		// add reference to commentID to "users" bucket
-		userBkt, e := b.getUserBucket(tx, comment.User.ID)
-		if e != nil {
-			return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
-		}
-		// put into individual user's bucket with ts as a key
-		if e = userBkt.Put(commentTs, ref); e != nil {
-			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
-		}
-
-		// set info with the count for post url
-		if _, e = b.setInfo(tx, comment); e != nil {
-			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
-		}
-		return nil
-	})
-
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		return bucket.ForEach(func(k, v []byte) error {
-			comment := store.Comment{}
-			if e = json.Unmarshal(v, &comment); e != nil {
-				return errors.Wrap(e, "failed to unmarshal")
-			}
-			comments = append(comments, comment)
-			return nil
-		})
-	})
-
-	comments = SortComments(comments, sortFld)
-	return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		c := lastBkt.Cursor()
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			url, commentID, e := b.parseRef(v)
-			if e != nil {
-				return e
-			}
-			postBkt, e := b.getPostBucket(tx, url)
-			if e != nil {
-				return e
-			}
-
-			comment := store.Comment{}
-			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
-				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
-				continue
-			}
-			if comment.Deleted {
-				continue
-			}
-			comments = append(comments, comment)
-			if len(comments) >= max {
-				break
-			}
-		}
-		return nil
-	})
-
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return 0, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		var e error
-		count, e = b.count(tx, locator.URL, 0)
-		return e
-	})
-
-	return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		postsBkt := tx.Bucket([]byte(postsBucketName))
-
-		c := postsBkt.Cursor()
-		n := 0
-		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			n++
-			if skip > 0 && n <= skip {
-				continue
-			}
-			postURL := string(k)
-			infoBkt := tx.Bucket([]byte(infoBucketName))
-			info := store.PostInfo{}
-			if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
-				return errors.Wrapf(e, "can't load info for %s", postURL)
-			}
-			list = append(list, info)
-			if limit > 0 && len(list) >= limit {
-				break
-			}
-		}
-		return nil
-	})
-
-	return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return store.PostInfo{}, err
-	}
-
-	info := store.PostInfo{}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		infoBkt := tx.Bucket([]byte(infoBucketName))
-		if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
-			return errors.Wrapf(e, "can't load info for %s", locator.URL)
-		}
-		return nil
-	})
-
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if b.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-	commentRefs := []string{}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	if limit == 0 || limit > userLimit {
-		limit = userLimit
-	}
-
-	// get list of references to comments
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-
-		c := userIDBkt.Cursor()
-		skipComments := 0
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			if len(commentRefs) >= limit {
-				break
-			}
-			if skip > 0 && skipComments < skip {
-				skipComments++
-				continue
-			}
-			commentRefs = append(commentRefs, string(v))
-		}
-		return nil
-	})
-
-	if err != nil {
-		return comments, err
-	}
-
-	// retrieve comments for refs
-	for _, v := range commentRefs {
-		url, commentID, errParse := b.parseRef([]byte(v))
-		if errParse != nil {
-			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
-		}
-		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
-			comments = append(comments, c)
-		}
-	}
-
-	return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return 0, err
-	}
-	count := 0
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-		stats := userIDBkt.Stats()
-		count = stats.KeyN
-		return nil
-	})
-	return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return comment, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.load(bucket, []byte(commentID), &comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
-	if curComment, err := b.Get(locator, comment.ID); err == nil {
-		// preserve immutable fields
-		comment.ParentID = curComment.ParentID
-		comment.Locator = curComment.Locator
-		comment.Timestamp = curComment.Timestamp
-		comment.User = curComment.User
-	}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.save(bucket, []byte(comment.ID), comment)
-	})
-}
-
-// Close boltdb store
-func (b *BoltDB) Close() error {
-	errs := new(multierror.Error)
-	for site, db := range b.dbs {
-		err := errors.Wrapf(db.Close(), "can't close site %s", site)
-		errs = multierror.Append(errs, err)
-	}
-	return errs.ErrorOrNil()
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res := postsBkt.Bucket([]byte(postURL))
-	if res == nil {
-		return nil, errors.Errorf("no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
-	if err != nil {
-		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
-	usersBkt := tx.Bucket([]byte(userBucketName))
-	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
-	if e != nil {
-		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
-	}
-	return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
-	if value == nil {
-		return errors.Errorf("can't save nil value for %s", key)
-	}
-	jdata, jerr := json.Marshal(value)
-	if jerr != nil {
-		return errors.Wrap(jerr, "can't marshal comment")
-	}
-	if err = bkt.Put(key, jdata); err != nil {
-		return errors.Wrapf(err, "failed to save key %s", key)
-	}
-	return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
-	value := bkt.Get(key)
-	if value == nil {
-		return errors.Errorf("no value for %s", key)
-	}
-
-	if err := json.Unmarshal(value, &res); err != nil {
-		return errors.Wrap(err, "failed to unmarshal")
-	}
-	return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
-		info = store.PostInfo{}
-	}
-	if val == 0 { // get current count, don't update
-		return info.Count, nil
-	}
-	info.Count += val
-
-	return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
-		info = store.PostInfo{
-			Count:   0,
-			URL:     comment.Locator.URL,
-			FirstTS: comment.Timestamp,
-			LastTS:  comment.Timestamp,
-		}
-	}
-	info.Count++
-	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
-	if res, ok := b.dbs[siteID]; ok {
-		return res, nil
-	}
-	return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
-	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
-	elems := strings.Split(string(val), "!!")
-	if len(elems) != 2 {
-		return "", "", errors.Errorf("invalid reference value %s", string(val))
-	}
-	return elems[0], elems[1], nil
-}

backend/app/store/engine/bolt_accessor_test.go

@@ -1,384 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
-	assert.Equal(t, "key id-1 already in store", err.Error())
-
-	_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-
-	assert.NoError(t, b.Close())
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	comment := store.Comment{
-		ID:        "id-ro",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	err := b.SetReadOnly(comment.Locator, true)
-	require.Nil(t, err)
-
-	_, err = b.Create(comment)
-	assert.NotNil(t, err)
-	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
-	err = b.SetReadOnly(comment.Locator, false)
-	require.Nil(t, err)
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err)
-
-	_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = b.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = b.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = b.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	_, err = b.Last("bad", 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Count(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-
-	_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
-	b, teardown := prep(t) // two comments for https://radio-t.com
-	defer teardown()
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
-	res, err := b.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = b.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = b.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = b.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	_, err = b.List("bad", 1, 1)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
-	b, teardown := prep(t) // two comments for https://radio-t.com
-	defer teardown()
-
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	_, err = b.User("bad", "user1", 1, 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.User("radio-t", "userZ", 1, 0)
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
-	os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-
-	defer func() {
-		require.NoError(t, b.Close())
-		os.Remove(testDb)
-	}()
-
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err = b.Create(c)
-		require.Nil(t, err)
-	}
-
-	// get all comments
-	res, err := b.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = b.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	count, err := b.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	_, err = b.UserCount("bad", "user1")
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.UserCount("radio-t", "userZ")
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
-	b := BoltDB{}
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	res := b.makeRef(comment)
-	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
-	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/2", url)
-	assert.Equal(t, "12345", id)
-
-	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
-	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
-	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) (b *BoltDB, teardown func()) {
-	os.Remove(testDb)
-
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b = boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	teardown = func() {
-		require.NoError(t, b.Close())
-		os.Remove(testDb)
-	}
-	return b, teardown
-}

backend/app/store/engine/bolt_admin.go

@@ -1,339 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		comment := store.Comment{}
-		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
-			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
-		}
-		// set deleted status and clear fields
-		comment.SetDeleted(mode)
-
-		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
-			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
-		}
-
-		// delete from "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err = lastBkt.Delete([]byte(commentID)); err != nil {
-			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
-		}
-
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
-		return nil
-	})
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
-	// delete top-level buckets
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		for _, bktName := range toDelete {
-
-			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
-			}
-			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-			}
-		}
-		return nil
-	})
-
-	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// get list of all comments outside of transaction loop
-	posts, err := b.List(siteID, 0, 0)
-	if err != nil {
-		return err
-	}
-
-	type commentInfo struct {
-		locator   store.Locator
-		commentID string
-	}
-
-	// get list of commentID for all user's comment
-	comments := []commentInfo{}
-	for _, postInfo := range posts {
-		err = bdb.View(func(tx *bolt.Tx) error {
-			postsBkt := tx.Bucket([]byte(postsBucketName))
-			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
-				comment := store.Comment{}
-				if err = json.Unmarshal(commentVal, &comment); err != nil {
-					return errors.Wrap(err, "failed to unmarshal")
-				}
-				if comment.User.ID == userID {
-					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
-				}
-				return nil
-			})
-			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
-	// delete collected comments
-	for _, ci := range comments {
-		if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
-			return errors.Wrapf(err, "failed to delete comment %+v", ci)
-		}
-	}
-
-	//  delete  user bucket
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		if usersBkt != nil {
-			if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
-				return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "can't delete user meta")
-	}
-
-	if len(comments) == 0 {
-		return errors.Errorf("unknown user %s", userID)
-	}
-
-	return err
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		switch status {
-		case true:
-			val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
-			if ttl > 0 {
-				val = time.Now().Add(ttl).Format(tsNano)
-			}
-			if e := bucket.Put([]byte(userID), []byte(val)); e != nil {
-				return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
-			}
-		}
-		return nil
-	})
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		val := bucket.Get([]byte(userID))
-		if val == nil {
-			blocked = false
-			return nil
-		}
-
-		until, e := time.Parse(tsNano, string(val))
-		if e != nil {
-			blocked = false
-			return nil
-		}
-		blocked = time.Now().Before(until)
-		return nil
-	})
-	return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
-			if errParse != nil {
-				return errors.Wrap(errParse, "can't parse block ts")
-			}
-			if time.Now().Before(ts) {
-				// get user name from comment user section
-				userName := ""
-				userComments, errUser := b.User(siteID, string(k), 1, 0)
-				if errUser == nil && len(userComments) > 0 {
-					userName = userComments[0].User.Name
-				}
-				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
-			}
-			return nil
-		})
-	})
-
-	return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
-			}
-		case false:
-			if e := bucket.Delete([]byte(locator.URL)); e != nil {
-				return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
-			}
-		}
-		return nil
-	})
-}
-
-// IsReadOnly checks if post in RO mode
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		ro = bucket.Get([]byte(locator.URL)) != nil
-		return nil
-	})
-	return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set verified status for %s", userID)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean verified status for %s", userID)
-			}
-		}
-		return nil
-	})
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		verified = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return verified
-}
-
-// Verified returns list of verified userIDs
-func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(verifiedBucketName))
-		_ = usersBkt.ForEach(func(k, _ []byte) error {
-			ids = append(ids, string(k))
-			return nil
-		})
-		return nil
-	})
-	return ids, err
-}

backend/app/store/engine/bolt_admin_test.go

@@ -1,246 +0,0 @@
-package engine
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	count, err := b.Count(loc)
-	require.NoError(t, err)
-	assert.Equal(t, 2, count, "count=2 initially")
-
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	count, err = b.Count(loc)
-	require.NoError(t, err)
-	assert.Equal(t, 1, count)
-
-	err = b.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	err = b.DeleteAll("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	err := b.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	_, err = b.User("radio-t", "user1", 5, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	err = b.DeleteUser("radio-t-bad", "user1")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0))
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
-	assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0))
-
-	assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-	time.Sleep(50 * time.Millisecond)
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
-	assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0))
-
-	ids, err := b.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	time.Sleep(50 * time.Millisecond)
-	ids, err = b.Blocked("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-
-	_, err = b.Blocked("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, b.SetVerified("radio-t", "u1", false))
-	assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u2", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u3", false))
-
-	ids, err := b.Verified("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
-	_, err = b.Verified("radio-t-bad")
-	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
-}

backend/app/store/engine/bolt_test.go

@@ -0,0 +1,934 @@
+package engine
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestBoltDB_CreateAndFind(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	var bb Interface = b
+	_ = bb
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+	t.Log(res[0].ID)
+
+	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	assert.Error(t, err)
+	assert.Equal(t, "key id-1 already in store", err.Error())
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+	assert.NoError(t, b.Close())
+}
+
+func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestBoltDB_Get(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.Error(t, err)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_Update(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	comment.Locator.SiteID = "radio-t"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
+}
+
+func TestBoltDB_FindLast(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_FindLastSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindInPostSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindForUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `no comments for user userZ in store`)
+}
+
+func TestBoltDB_FindForUserPagination(t *testing.T) {
+	_ = os.Remove(testDb)
+	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.NoError(t, err)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("id-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err = b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+	assert.Equal(t, "id-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "id-189", res[0].ID)
+	assert.Equal(t, "id-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "id-4", res[0].ID)
+	assert.Equal(t, "id-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_CountPost(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_CountUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`)
+}
+
+func TestBoltDB_InfoPost(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestBoltDB_InfoList(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
+
+	req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	_, err = b.Info(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_FlagBlockedUser(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err, "non-existing user can't be blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked on wrong site")
+}
+
+func TestBolt_FlagReadOnlyPost(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.Equal(t, true, val)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro on wrong site")
+}
+
+func TestBolt_FlagVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	isVerified := func(site, user string) bool {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`)
+	assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", FlagFalse))
+}
+
+func TestBolt_FlagListVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	ids, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	_, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
+}
+
+func TestBolt_FlagListBlocked(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error {
+		req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 150*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
+
+	vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	require.Equal(t, 2, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+	assert.Equal(t, "user2", blockedList[1].ID)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(150 * time.Millisecond)
+	vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	_, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.EqualError(t, err, `site "bad" not found`)
+
+}
+
+func TestBoltDB_UserDetail(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	// add two entries to DB before we start
+	result, err := b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = b.UserDetail(UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u2", Detail: UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      UserDetailRequest
+		error    string
+		expected []UserDetailEntry
+	}{
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1", Detail: UserEmail},
+			expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: UserEmail},
+			error: `site "bad" not found`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "u1xyz", Detail: UserEmail}},
+		{req: UserDetailRequest{Detail: UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: UserDetailRequest{Detail: UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: UserDetailRequest{Update: "not_relevant", Detail: AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, Detail: AllUserDetails},
+			error: `site "bad" not found`},
+		{req: UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, Detail: AllUserDetails},
+			expected: []UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := b.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestBolt_DeleteComment(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 2, count, "count=2 initially")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
+}
+
+func TestBolt_DeleteHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestBolt_DeleteAll(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail, Update: "value1"}
+		readUser   = UserDetailRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Detail: UserEmail}
+		emailSet   = []UserDetailEntry{{UserID: "user1", Email: "value1"}}
+	)
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	var testData = []struct {
+		delReq    DeleteRequest
+		detailReq UserDetailRequest
+		expected  []UserDetailEntry
+		err       string
+	}{
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: readUser, expected: emailSet, err: `site "bad" not found`},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: UserEmail},
+			detailReq: readUser},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", UserDetail: AllUserDetails},
+			detailReq: readUser},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		if x.err == "" {
+			require.NoError(t, err, "delete request #%d error", i)
+		} else {
+			require.EqualError(t, err, x.err, "delete request #%d error", i)
+		}
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestBoltAdmin_DeleteUserHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.EqualError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltDB_ref(t *testing.T) {
+	b := BoltDB{}
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	res := b.makeRef(comment)
+	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
+
+	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
+	assert.NoError(t, err)
+	assert.Equal(t, "https://radio-t.com/2", url)
+	assert.Equal(t, "12345", id)
+
+	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
+	assert.Error(t, err)
+}
+
+func TestBoltDB_NewFailed(t *testing.T) {
+	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
+	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
+	_ = os.Remove(testDb)
+
+	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	assert.NoError(t, err)
+	b = boltStore
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+
+	teardown = func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
+	return b, teardown
+}
+
+func getReq(locator store.Locator, commentID string) GetRequest {
+	return GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/app/store/engine/engine.go

@@ -1,6 +1,7 @@
+package engine
+
 // Package engine defines interfaces each supported storage should implement.
 // Includes default implementation with boltdb
-package engine
 
 import (
 	"sort"
@@ -10,50 +11,108 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
-//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go"
+// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
+//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
 
-// Interface combines all store interfaces
+// Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Accessor
-	Admin
+	Create(comment store.Comment) (commentID string, err error)  // create new comment, avoid dups by id
+	Update(comment store.Comment) error                          // update comment, mutable parts only
+	Get(req GetRequest) (store.Comment, error)                   // get comment by id
+	Find(req FindRequest) ([]store.Comment, error)               // find comments for locator or site
+	Info(req InfoRequest) ([]store.PostInfo, error)              // get post(s) meta info
+	Count(req FindRequest) (int, error)                          // get count for post or user
+	Delete(req DeleteRequest) error                              // Delete post(s), user, comment, user details, or everything
+	Flag(req FlagRequest) (bool, error)                          // set and get flags
+	ListFlags(req FlagRequest) ([]interface{}, error)            // get list of flagged keys, like blocked & verified user
+	UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) // sets or gets single detail value, or gets all details for requested site.
+	// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+	// and all site's details listing under the same function (and not to extend interface by two separate functions).
+	Close() error // close storage engine
 }
 
-// UserRequest is the request send to get comments by user
-type UserRequest struct {
-	SiteID string
-	UserID string
-	Limit  int
-	Skip   int
+// GetRequest is the input for Get func
+type GetRequest struct {
+	Locator   store.Locator `json:"locator"`
+	CommentID string        `json:"comment_id"`
 }
 
-// Accessor defines all usual access ops avail for regular user
-type Accessor interface {
-	Create(comment store.Comment) (commentID string, err error)           // create new comment, avoid dups by id
-	Get(locator store.Locator, commentID string) (store.Comment, error)   // get comment by id
-	Put(locator store.Locator, comment store.Comment) error               // update comment, mutable parts only
-	Find(locator store.Locator, sort string) ([]store.Comment, error)     // find comments for locator
-	Last(siteID string, limit int) ([]store.Comment, error)               // last comments for given site, sorted by time
-	User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
-	UserCount(siteID, userID string) (int, error)                         // comments count by user
-	Count(locator store.Locator) (int, error)                             // number of comments for the post
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)    // list of commented posts
-	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)  // get post info
-	Close() error                                                         // close/stop engine
+// FindRequest is the input for all find operations
+type FindRequest struct {
+	Locator store.Locator `json:"locator"`           // lack of URL means site operation
+	UserID  string        `json:"user_id,omitempty"` // presence of UserID treated as user-related find
+	Sort    string        `json:"sort,omitempty"`    // sort order with +/-field syntax
+	Since   time.Time     `json:"since,omitempty"`   // time limit for found results
+	Limit   int           `json:"limit,omitempty"`
+	Skip    int           `json:"skip,omitempty"`
 }
 
-// Admin defines all store ops avail for admin only
-type Admin interface {
-	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
-	DeleteAll(siteID string) error                                               // delete all data from site
-	DeleteUser(siteID string, userID string) error                               // remove all comments from user
-	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent)
-	IsBlocked(siteID string, userID string) bool                                 // check if user blocked
-	Blocked(siteID string) ([]store.BlockedUser, error)                          // get list of blocked users
-	SetReadOnly(locator store.Locator, status bool) error                        // set/reset read-only flag
-	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
-	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
-	IsVerified(siteID string, userID string) bool                                // check verified status
-	Verified(siteID string) ([]string, error)                                    // list of verified user ids
+// InfoRequest is the input of Info operation used to get meta data about posts
+type InfoRequest struct {
+	Locator     store.Locator `json:"locator"`
+	Limit       int           `json:"limit,omitempty"`
+	Skip        int           `json:"skip,omitempty"`
+	ReadOnlyAge int           `json:"ro_age,omitempty"`
+}
+
+// DeleteRequest is the input for all delete operations (comments, sites, users)
+type DeleteRequest struct {
+	Locator    store.Locator    `json:"locator"` // lack of URL means site operation
+	CommentID  string           `json:"comment_id,omitempty"`
+	UserID     string           `json:"user_id,omitempty"`
+	UserDetail UserDetail       `json:"user_detail,omitempty"`
+	DeleteMode store.DeleteMode `json:"del_mode"`
+}
+
+// Flag defines type of binary attribute
+type Flag string
+
+// FlagStatus represents values of the flag update
+type FlagStatus int
+
+// enum of update values
+const (
+	FlagNonSet FlagStatus = 0
+	FlagTrue   FlagStatus = 1
+	FlagFalse  FlagStatus = -1
+)
+
+// Enum of all flags
+const (
+	ReadOnly = Flag("readonly")
+	Verified = Flag("verified")
+	Blocked  = Flag("blocked")
+)
+const (
+	// All possible user details
+	UserEmail      = UserDetail("email")
+	AllUserDetails = UserDetail("all") // used for listing and deletion requests
+)
+
+// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
+type FlagRequest struct {
+	Flag    Flag          `json:"flag"`              // flag type
+	Locator store.Locator `json:"locator"`           // post locator
+	UserID  string        `json:"user_id,omitempty"` // for flags setting user status
+	Update  FlagStatus    `json:"update,omitempty"`  // if FlagNonSet it will be get op, if set will set the value
+	TTL     time.Duration `json:"ttl,omitempty"`     // ttl for time-sensitive flags only, like blocked for some period
+}
+
+// UserDetail defines name of the user detail
+type UserDetail string
+
+// UserDetailEntry contains single user details entry
+type UserDetailEntry struct {
+	UserID string `json:"user_id"`         // duplicate user's id to use this structure not only embedded but separately
+	Email  string `json:"email,omitempty"` // UserEmail
+}
+
+// UserDetailRequest is the input for both get/set for details, like email
+type UserDetailRequest struct {
+	Detail  UserDetail    `json:"detail"`           // detail name
+	Locator store.Locator `json:"locator"`          // post locator
+	UserID  string        `json:"user_id"`          // user id for get\set
+	Update  string        `json:"update,omitempty"` // update value
 }
 
 const (

backend/app/store/engine/engine_mock.go

@@ -0,0 +1,231 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package engine
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	store "github.com/umputun/remark/backend/app/store"
+)
+
+// MockInterface is an autogenerated mock type for the Interface type
+type MockInterface struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *MockInterface) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Count provides a mock function with given fields: req
+func (_m *MockInterface) Count(req FindRequest) (int, error) {
+	ret := _m.Called(req)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(FindRequest) int); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Create provides a mock function with given fields: comment
+func (_m *MockInterface) Create(comment store.Comment) (string, error) {
+	ret := _m.Called(comment)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(store.Comment) string); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(store.Comment) error); ok {
+		r1 = rf(comment)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Delete provides a mock function with given fields: req
+func (_m *MockInterface) Delete(req DeleteRequest) error {
+	ret := _m.Called(req)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Find provides a mock function with given fields: req
+func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.Comment
+	if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.Comment)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Flag provides a mock function with given fields: req
+func (_m *MockInterface) Flag(req FlagRequest) (bool, error) {
+	ret := _m.Called(req)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Get provides a mock function with given fields: req
+func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 store.Comment
+	if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(store.Comment)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(GetRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Info provides a mock function with given fields: req
+func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.PostInfo
+	if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.PostInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(InfoRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListFlags provides a mock function with given fields: req
+func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) {
+	ret := _m.Called(req)
+
+	var r0 []interface{}
+	if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]interface{})
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Update provides a mock function with given fields: comment
+func (_m *MockInterface) Update(comment store.Comment) error {
+	ret := _m.Called(comment)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(store.Comment) error); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// UserDetail provides a mock function with given fields: req
+func (_m *MockInterface) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
+	ret := _m.Called(req)
+
+	var r0 []UserDetailEntry
+	if rf, ok := ret.Get(0).(func(UserDetailRequest) []UserDetailEntry); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]UserDetailEntry)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(UserDetailRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

backend/app/store/engine/mongo.go

@@ -1,377 +0,0 @@
-package engine
-
-import (
-	"time"
-
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	"github.com/go-pkgz/mongo"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Mongo implements engine interface
-type Mongo struct {
-	conn       *mongo.Connection
-	postWriter mongo.BufferedWriter
-}
-
-const (
-	mongoPosts     = "posts"
-	mongoMetaPosts = "meta_posts"
-	mongoMetaUsers = "meta_users"
-)
-
-type metaPost struct {
-	ID       string `bson:"_id"` // url
-	SiteID   string `bson:"site"`
-	ReadOnly bool   `bson:"read_only"`
-}
-
-type metaUser struct {
-	ID           string    `bson:"_id"` // user_id
-	SiteID       string    `bson:"site"`
-	Verified     bool      `bson:"verified"`
-	Blocked      bool      `bson:"blocked"`
-	BlockedUntil time.Time `bson:"blocked_until"`
-}
-
-// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
-// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
-func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
-	writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
-	result := Mongo{conn: conn, postWriter: writer}
-	err := result.prepare()
-	return &result, errors.Wrap(err, "failed to prepare mongo")
-}
-
-// Create new comment, write can be buffered and delayed.
-func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
-	// err = m.postWriter.Write(comment)
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Insert(&comment)
-	})
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).Sort(sortFld).All(&comments)
-	})
-	return comments, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).One(&comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
-			bson.M{"$set": bson.M{
-				"text":    comment.Text,
-				"orig":    comment.Orig,
-				"score":   comment.Score,
-				"votes":   comment.Votes,
-				"pin":     comment.Pin,
-				"deleted": comment.Deleted,
-			}})
-	})
-}
-
-// Last returns up to max last comments for given siteID
-func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "delete": false}
-		return coll.Find(query).Sort("-time").Limit(max).All(&comments)
-	})
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (m *Mongo) Count(locator store.Locator) (count int, err error) {
-
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
-		count, err = coll.Find(query).Count()
-		return err
-	})
-	return count, e
-}
-
-// List returns list of all commented posts with counters
-func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-	list = []store.PostInfo{}
-
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": siteID}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-			{"$skip": skip},
-			{"$limit": limit},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	return list, errors.Wrap(err, "can't get list")
-}
-
-// Info returns time range and count for locator
-func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
-	list := []store.PostInfo{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	if err != nil {
-		return info, err
-	}
-	if len(list) == 0 {
-		return info, errors.Errorf("can't load info for %s", locator.URL)
-	}
-	info = list[0]
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if m.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, nil
-}
-
-// User extracts all comments for given site and given userID
-func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "user.id": userID}
-		return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
-	})
-	return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
-}
-
-// UserCount returns number of comments for user
-func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		var e error
-		count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
-		return e
-	})
-	return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
-	return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
-		return e
-	})
-}
-
-// IsReadOnly checks if post in RO
-func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
-	meta := metaPost{}
-	err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
-	})
-	return err == nil && meta.ReadOnly
-}
-
-// SetVerified makes user verified or reset the flag
-func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
-		return e
-	})
-}
-
-// IsVerified checks if user verified
-func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Verified
-}
-
-// Verified returns list of verified user IDs
-func (m *Mongo) Verified(siteID string) (ids []string, err error) {
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
-	})
-	if err != nil {
-		return nil, err
-	}
-	for _, meta := range metas {
-		ids = append(ids, meta.ID)
-	}
-	return ids, nil
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	until := time.Time{}
-	if status {
-		until = time.Now().AddDate(100, 0, 0) // permanent is 50year
-		if ttl > 0 {
-			until = time.Now().Add(ttl)
-		}
-	}
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
-			bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
-		return errors.Wrapf(e, "failed to set block for %s", userID)
-	})
-}
-
-// IsBlocked checks if user blocked
-func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
-}
-
-// Blocked get lists of blocked users for given site
-func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID,
-			"blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
-	})
-	if err != nil {
-		return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
-	}
-
-	for _, mu := range metas {
-		blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
-		if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
-			blockedUser.Name = ucc[0].User.Name
-		}
-		users = append(users, blockedUser)
-	}
-	return users, nil
-}
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-	comment := store.Comment{}
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
-		if e != nil {
-			return e
-		}
-		comment.SetDeleted(mode)
-		return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
-	})
-	return errors.Wrapf(err, "can't delete %s", commentID)
-}
-
-// DeleteAll removes all info about siteID
-func (m *Mongo) DeleteAll(siteID string) error {
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		_, e := coll.RemoveAll(bson.M{"locator.site": siteID})
-		return e
-	})
-	return errors.Wrapf(err, "can't delete site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted".
-func (m *Mongo) DeleteUser(siteID string, userID string) error {
-	comments := []store.Comment{}
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
-		if e != nil {
-			return e
-		}
-		for _, c := range comments {
-			if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
-				return e
-			}
-		}
-		return nil
-	})
-}
-
-// Close boltdb store
-func (m *Mongo) Close() error {
-	if m.postWriter != nil {
-		return m.postWriter.Close()
-	}
-	return nil
-}
-
-// prepare collections with all indexes
-func (m *Mongo) prepare() error {
-	errs := new(multierror.Error)
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
-	})
-}
-
-func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-	return q.Skip(skip).Limit(limit)
-}