backport url sanitizer to 1.6

change golangci-lint format to github-actions for CI

.dockerignore

@@ -20,4 +20,5 @@ debug.test
 *.prof
 *.test
 remark42
-/backend/var/
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -1,101 +1,103 @@
-workspace:
-  base: /go
-  path: src/github.com/umputun/remark
+kind: pipeline
+name: default
+type: docker
 
-pipeline:
-  
-  build:
-    image: golang:1.12-alpine
+steps:
+
+  - name: build server
+    image: umputun/baseimage:buildgo-latest
     commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
       - cd backend/app
-      - go build -v ./...
+      - go build -mod=vendor
+      - echo "build completed"
 
-  docker_master:
+  - name: docker master
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
     when:
-      branch: [master, release/*]
+      branch: [master]
       event: push
 
-  docker_tag:
+  - name: docker tag
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-    tags: 
-      - ${DRONE_TAG}
-      - latest
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+      tags:
+        - ${DRONE_TAG}
+        - latest
     when:
       event: tag
 
-  artifacts_tag:
+  - name: docker branch
     image: plugins/docker
-    dockerfile: Dockerfile.artifacts
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
-    when:
-      event: tag
-
-  docker_branch:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
+      dry_run: true
     when:
       branch:
         exclude: [master, release/*]
         event: push
 
-  deploy:
-    image: appleboy/drone-ssh
-    host: remark42.com
-    username: umputun
-    port: 22
-    secrets: [ ssh_key ]
-    script:
-      - cd /srv/remark
-      - docker-compose pull
-      - docker-compose up -d
+  - name: artifacts tag
+    image: plugins/docker
+    settings:
+      dockerfile: Dockerfile.artifacts
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - GITHUB_TOKEN=${GITHUB_TOKEN}
+    when:
+      event: tag
+
+  - name: deploy
+    image: docker.umputun.com/system/deploy-ci:master
+    commands:
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
     when:
       branch: master
       event: push
 
-  notify:
+  - name: notify
     image: drillster/drone-email
-    host: smtp.mailgun.org
-    port: 25
-    username: email_username
-    password: email_password
-    from: drone@mg.umputun.com
-    recipients: [ umputun@gmail.com ]
-    secrets: [ email_username, email_password ]
+    settings:
+      host: smtp.mailgun.org
+      username:
+        from_secret: email_username
+      password:
+        from_secret: email_password
+      from: drone@mg.umputun.dev
+      recipients: [ sys@umputun.dev ]
     when:
       status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*       @umputun

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+patreon: remark42

.github/workflows/ci-build.yml

@@ -0,0 +1,31 @@
+name: build
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: build docker image
+        run: docker build --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github .

.github/workflows/ci-test-backend.yml

@@ -0,0 +1,63 @@
+name: test_backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: debug if needed
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.14
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.25.0
+          go get -u github.com/mattn/goveralls
+
+      - name: test and lint backend
+        run: |
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          GOFLAGS: "-mod=vendor"
+          TZ: "America/Chicago"
+
+      - name: test and lint examples
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
+
+      - name: submit coverage
+        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-test-frontend.yml

@@ -0,0 +1,28 @@
+name: test_frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-frontend.yml'
+      - 'frontend/**'
+
+jobs:
+  frontend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npx run-p check lint
+        working-directory: ./frontend

.gitignore

@@ -19,3 +19,8 @@ debug.test
 remark42
 /bin/
 /backend/var/
+/backend/app/var/
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor

.travis.yml

@@ -1,25 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
-      .
-  - docker rm -f mongo
-  

Dockerfile

@@ -1,17 +1,6 @@
 FROM umputun/baseimage:buildgo-latest as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
 ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
@@ -19,44 +8,29 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
+ARG BACKEND_TEST_TIMEOUT
 
 ADD backend /build/backend
-ADD .git /build/.git
+ADD .git/ /build/backend/.git/
 WORKDIR /build/backend
 
+ENV GOFLAGS="-mod=vendor"
+
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
-    else echo "skip backend test" ; fi
-
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
-# linters
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-        --enable=ineffassign --enable=varcheck ./... ; \
-    else echo "skip backend linters" ; fi
-
-# submit coverage to coverals if COVERALLS_TOKEN in env
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo "coverall not enabled" ; \
-    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+        golangci-lint run --config ../.golangci.yml ./... ; \
+    else echo "skip backend tests and linter" ; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
-    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
     echo "version=$version" && \
-    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
-
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 FROM node:10.11-alpine as build-frontend-deps
 
@@ -66,7 +40,7 @@ ENV HUSKY_SKIP_INSTALL=true
 RUN apk add --no-cache --update git
 ADD frontend/package.json /srv/frontend/package.json
 ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/frontend && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci --loglevel warn
 
 FROM node:10.11-alpine as build-frontend
 
@@ -77,16 +51,15 @@ ARG NODE_ENV=production
 COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
 ADD frontend /srv/frontend
 RUN cd /srv/frontend && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-
-FROM umputun/baseimage:app-latest
+FROM umputun/baseimage:app
 
 WORKDIR /srv
 
-ADD entrypoint.sh /entrypoint.sh
+ADD docker-init.sh /entrypoint.sh
 ADD backend/scripts/backup.sh /usr/local/bin/backup
 ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
@@ -100,5 +73,6 @@ RUN ln -s /srv/remark42 /usr/bin/remark42
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["server"]
-ENTRYPOINT ["/entrypoint.sh"]
+COPY docker-init.sh /srv/init.sh
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -26,72 +26,80 @@ RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
-
 FROM umputun/baseimage:buildgo-latest as build-backend
 
 ARG GITHUB_TOKEN
 ENV SKIP_BACKEND_TEST=true
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD README.md /go/src/github.com/umputun/remark/
-ADD LICENSE /go/src/github.com/umputun/remark/
+RUN go get github.com/rakyll/statik
+
+WORKDIR /build/backend
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+ADD .git/ /build/backend/.git/
+
 COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
-    export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
+    export WEB_ROOT=/build/backend/web && \
     sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
-    sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
-    go get -v github.com/rakyll/statik && \
-    statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
-    ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
-    ls -la /go/src/github.com/umputun/remark/backend/web/
+    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
+    ls -la /build/backend/app/rest/api/statik.go && \
+    ls -la /build/backend/web/
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
     if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    echo "runs outside of drone" && version=$(/script/git-rev.sh); \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
+    export GOFLAGS="-mod=vendor" && \
     GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
-    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
 
 RUN \
     if [ -z "$DRONE_TAG" ] ; then \
     echo "runs outside of drone" && tag=""; \
     else tag=_${DRONE_TAG}; fi && \
     apk add --no-cache --update zip && \
-    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
-    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
 # upload to github
-RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "skip upload to github" ; \
-    else \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+#RUN \
+#    if [ -z "$DRONE_TAG" ] ; then \
+#    echo "skip upload to github" ; \
+#    else \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
 RUN ls -la /artifacts/*
 CMD ["sleep", "100"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2020 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -20,7 +20,21 @@ deploy:
 	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
 	docker rm -f remark42.bin
 
-.PHONY: bin
+race_test:
+	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,11 +1,14 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# remark42 [![Build Status](https://github.com/umputun/remark/workflows/build/badge.svg)](https://github.com/umputun/remark/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+
+
 
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, GitHub and Yandex
+* Login via email
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus and wordpress
+* Import from Disqus and WordPress
 * Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
@@ -13,7 +16,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
-* Telegram notifications
+* Telegram and email notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
@@ -23,13 +26,23 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
+
+<details><summary>Screenshots</summary>
+
+Comments example:
+![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
+
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
+</details>
+
 
 #
 
   - [Install](#install)
     - [Backend](#backend)
       - [With Docker](#with-docker)
-      - [Without docker](#without-docker)
+      - [Without Docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
       - [Quick installation test](#quick-installation-test)
@@ -37,6 +50,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
         - [Facebook Auth Provider](#facebook-auth-provider)
+        - [Twitter Auth Provider](#twitter-auth-provider)
         - [Yandex Auth Provider](#yandex-auth-provider)
       - [Initial import from Disqus](#initial-import-from-disqus)
       - [Initial import from WordPress](#initial-import-from-wordpress)
@@ -75,93 +89,136 @@ _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
 * make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
-* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
+* pull prepared images from the DockerHub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
-#### Without docker
+#### Without Docker
 
 * download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
 * unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
 * run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
-* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
+* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
 
 #### Parameters
 
-| Command line            | Environment             | Default                  | Description                                      |
-| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
-| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                          | secret key, _required_                           |
-| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                          | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
-| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
-| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
-| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
-| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
-| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
-| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
-| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
-| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
-| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
-| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
-| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
-| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
-| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
-| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
-| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                     |
+| ----------------------- | ----------------------- | ------------------------ | ----------------------------------------------- |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_              |
+| secret                  | SECRET                  |                          | secret key, _required_                          |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                           |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `rpc`                |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                          |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                           |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_         |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                     |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                        |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited     |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited       |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited  |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `uri`  |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                 |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                     |
+| avatar.uri              | AVATAR_URI              | `./var/avatars`          | avatar store uri                                |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save     |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, `bolt`             |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                          |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                    |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                      |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                      |
+| image.bolt.file         | IMAGE_BOLT_FILE         | `/var/pictures.db`       | images bolt file location                       |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `2400`                   | width of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `900`                    | height of resized image                         |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                         |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                      |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                          |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                      |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                        |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                    |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                          |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                      |
+| auth.twitter.cid        | AUTH_TWITTER_CID        |                          | Twitter Consumer API Key                        |
+| auth.twitter.csec       | AUTH_TWITTER_CSEC       |                          | Twitter Consumer API Secret key                 |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                          |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                      |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only      |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                          |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                           |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                      |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                   |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                              |
+| auth.email.template     | AUTH_EMAIL_TEMPLATE     | none (predefined)        | custom email message template file              |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (telegram and/or email)    |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                      |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                  |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
+| notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
+| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
+| notify.email.notify_admin | NOTIFY_EMAIL_ADMIN    | `false`                  | notify admin on new comments via ADMIN_SHARED_EMAIL |
+| smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
+| smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
+| smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
+| smtp.password           | SMTP_PASSWORD           |                          | SMTP password                                   |
+| smtp.tls                | SMTP_TLS                |                          | enable TLS for SMTP                             |
+| smtp.timeout            | SMTP_TIMEOUT            | `10s`                    | SMTP TCP connection timeout                     |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le  |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                           |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                           |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                            |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored      |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                            |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited       |
+| votes-ip                | VOTES_IP                | `false`                  | restrict votes from the same ip                 |
+| anon-vote               | ANON_VOTE               | `false`                  | allow voting for anonymous users, require VOTES_IP to be enabled as well |
+| votes-ip-time           | VOTES_IP_TIME           | `5m`                     | same ip vote restriction time, `0s` - unlimited |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                             |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                        |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive   |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_ |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                     |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                 |
+| image-proxy.http2https  |  IMAGE_PROXY_HTTP2HTTPS | `false`                  | enable http->https proxy for images             |
+| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false`            | enable caching external images to current image storage |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                            |
+| simple-view             | SIMPLE_VIEW             | `false`                  | minimized UI with basic info only               |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                 |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                       |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                               |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                 |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                      |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
 * _required_ parameters have to be presented in the environment or provided in command line
 
+##### Deprecated
+
+Following list of command-line options is deprecated and will be removed in 2 minor releases or 1 major release (whichever is closer)
+from the version in which they were deprecated. After remark42 version update, please check startup log once for deprecation warnings to avoid
+trouble with unrecognized command-line options in the future.
+
+<details>
+<summary>deprecated options</summary>
+
+| Command line       | Replacement   | Environment        | Replacement   | Default | Description    | Deprecation version |
+| ------------------ | ------------- | ------------------ | ------------- | ------- | -------------- | ------------------- |
+| auth.email.host    | smtp.host     | AUTH_EMAIL_HOST    | SMTP_HOST     |         | smtp host      | 1.5.0               |
+| auth.email.port    | smtp.port     | AUTH_EMAIL_PORT    | SMTP_PORT     |         | smtp port      | 1.5.0               |
+| auth.email.user    | smtp.username | AUTH_EMAIL_USER    | SMTP_USERNAME |         | smtp user name | 1.5.0               |
+| auth.email.passwd  | smtp.password | AUTH_EMAIL_PASSWD  | SMTP_PASSWORD |         | smtp password  | 1.5.0               |
+| auth.email.tls     | smtp.tls      | AUTH_EMAIL_TLS     | SMTP_TLS      | `false` | enable TLS     | 1.5.0               |
+| auth.email.timeout | smtp.timeout  | AUTH_EMAIL_TIMEOUT | SMTP_TIMEOUT  | `10s`   | smtp timeout   | 1.5.0               |
+| img-proxy          | image-proxy.http2https | IMG_PROXY | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images | 1.5.0 |
+</details>
+
 ##### Required parameters
 
 Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
-2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
+2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.remark42.com`
 3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
 
 The minimal `docker-compose.yml` has to include all required parameters:
@@ -227,6 +284,14 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
 1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
 
+##### Twitter Auth Provider
+
+1.	Create a new twitter application https://developer.twitter.com/en/apps
+1.	Fill **App name**, **Description** and **URL** of your site
+1.	In the field **Callback URLs** enter the correct url of your callback handler e.g.  domain + `/auth/twitter/callback`
+1.	Under **Key and tokens** take note of the **Consumer API Key** and **Consumer API Secret key**. Those will be used as `AUTH_TWITTER_CID` and
+ `AUTH_TWITTER_CSEC`
+
 ##### Yandex Auth Provider
 
 1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
@@ -297,6 +362,39 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
+#### Docker parameters
+
+Two parameters allow customizing Docker container on the system level:
+
+- `APP_UID` - sets UID to run remark42 application in container (default=1001)
+- `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
+
+_see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
+
+example of `docker-compose.yml`:
+
+```yaml
+version: '2'
+
+services:
+    remark42:
+        image: umputun/remark42:latest
+        restart: always
+        container_name: "remark42"
+        environment:
+            - APP_UID=2000                          # runs remark42 app with non-default UID
+            - TIME_ZONE=GTC                         # sets container time to UTC
+
+            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
+            - SECRET=abcd-123456-xyz-$%^&           # secret key
+            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
+            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
+        volumes:
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
+```
+
+
 ### Setup on your website
 
 #### Comments
@@ -308,18 +406,47 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
+    components: ['embed'], // optional param; which components to load. default to ["embed"]
+                           // to load all components define components as ['embed', 'last-comments', 'counter']
+                           // available component are:
+                           //     - 'embed': basic comments widget
+                           //     - 'last-comments': last comments widget, see `Last Comments` section below
+                           //     - 'counter': counter widget, see `Counter` section below
+    url: 'PAGE_URL', // optional param; if it isn't defined
+                     // `window.location.origin + window.location.pathname` will be used,
+                     //
+                     // Note that if you use query parameters as significant part of url
+                     // (the one that actually changes content on page)
+                     // you will have to configure url manually to keep query params, as
+                     // `window.location.origin + window.location.pathname` doesn't contain query params and
+                     // hash. For example default url for `https://example/com/example-post?id=1#hash`
+                     // would be `https://example/com/example-post`.
+                     //
+                     // The problem with query params is that they often contain useless params added by
+                     // various trackers (utm params) and doesn't have defined order, so Remark treats differently
+                     // all this examples:
+                     // https://example.com/?postid=1&date=2007-02-11
+                     // https://example.com/?date=2007-02-11&postid=1
+                     // https://example.com/?date=2007-02-11&postid=1&utm_source=google
+                     //
+                     // If you deal with query parameters make sure you pass only significant part of it
+                     // in well defined order
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
-    page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
+    page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
+    locale: 'en' // set up locale and language, if it isn't defined default value ('en') will be used
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -344,23 +471,35 @@ Just call this function and pass a name of the theme that you want to turn on:
 window.REMARK42.changeTheme('light');
 ```
 
+##### Locales
+
+Right now Remark is translated to en, ru (partially), de, and fi languages.
+You can pick one using [configuration object](#setup-on-your-website).
+
+Do you want translate remark42 to other locale? Please see [this documentation](https://github.com/umputun/remark42/blob/master/docs/translation.md) for details.
+
 #### Last comments
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -376,19 +515,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -402,17 +546,17 @@ You can use as many nodes like this as you need to.
 The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
+Also script can use `url` property from `remark_config` object, or `window.location.origin + window.location.pathname` if nothing else is defined.
 
 ## Build from the source
 
-- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
+- to build Docker container - `make docker`. This command will produce container `umputun/remark42`.
 - to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
  `remark42` file with everything embedded.
 
 ## Development
 
-You can use fully functional local version to develop and test both frontend & backend.
+You can use fully functional local version to develop and test both frontend & backend. It requires at least 2GB RAM or swap enabled
 
 To bring it up run:
 
@@ -430,32 +574,34 @@ To access UI demo page go to `127.0.0.1:8080/web`.
 By default, you would be logged in as `dev_user` which defined as admin.
 You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
 
-Backend docker compose config by default skips running frontend related tests.
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
+Backend Docker Compose config by default skips running frontend related tests.
+Frontend Docker Compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
 
 ### Backend development
 
-In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
+In order to run backend locally (development mode, without Docker) you have to have the latest stable `go` toolchain [installed](https://golang.org/doc/install).
 
-To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
+To run backend - `cd backend; go run app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
 It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
-`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
-
-To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
-`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
+`HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
 
 ### Frontend development
 
-#### Build
+#### Developer guide
 
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
+Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
+
+#### Build
+You should have at least 2GB RAM or swap enabled for building
+
+* install [Node.js 12.11](https://nodejs.org/en/) or higher;
+* install [NPM 6.13.4](https://www.npmjs.com/package/npm);
 * run `npm install` inside `./frontend`;
 * run `npm run build` there;
 * result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
-It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
+It used to reformat your frontend code using `prettier` and lint with `eslint` and `stylelint` before every commit.
 
 #### Devserver
 
@@ -470,12 +616,17 @@ You can attach to locally running backend by providing `REMARK_URL` environment
 npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 ```
 
+The best way for start local developer enviroment:
+```sh
+cp compose-dev-frontend.yml compose-private-frontend.yml
+docker-compose -f compose-private-frontend.yml up --build
+cd frontend
+npm run dev
+```
+
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
-#### Frontend guide
-
-Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
 
 ## API
 
@@ -561,7 +712,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
    }{}
 ```
 
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
 * `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
 * `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
   ```go
@@ -590,21 +741,58 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
   ```go
   type Config struct {
-      Version        string   `json:"version"`
-      EditDuration   int      `json:"edit_duration"`
-      MaxCommentSize int      `json:"max_comment_size"`
-      Admins         []string `json:"admins"`
-      AdminEmail     string   `json:"admin_email"`
-      Auth           []string `json:"auth_providers"`
-      LowScore       int      `json:"low_score"`
-      CriticalScore  int      `json:"critical_score"`
-      PositiveScore  bool     `json:"positive_score"`
-      ReadOnlyAge    int      `json:"readonly_age"`
-      MaxImageSize   int      `json:"max_image_size"`
+        Version        string   `json:"version"`
+        EditDuration   int      `json:"edit_duration"`
+        MaxCommentSize int      `json:"max_comment_size"`
+        Admins         []string `json:"admins"`
+        AdminEmail     string   `json:"admin_email"`
+        Auth           []string `json:"auth_providers"`
+        LowScore       int      `json:"low_score"`
+        CriticalScore  int      `json:"critical_score"`
+        PositiveScore  bool     `json:"positive_score"`
+        ReadOnlyAge    int      `json:"readonly_age"`
+        MaxImageSize   int      `json:"max_image_size"`
+        EmojiEnabled   bool     `json:"emoji_enabled"`
   }
   ```
 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
+* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
+
+### Streaming API
+
+Streaming API provide server-sent events for post updates as well as site update
+
+* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
+* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
+
+<details><summary>response example</summary>
+
+```
+data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}
+```
+
+</details>
 
 ### RSS feeds
 
@@ -619,6 +807,17 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
 _returned id should be appended to load image url on caller side_
 
+### Email subscription
+
+* `GET /api/v1/email?site=site-id` - get user's email, _auth required_
+* `POST /api/v1/email/subscribe?site=site-id&address=user@example.org` -  makes confirmation token and sends it to user over email, _auth required_
+
+  Trying to subscribe same email second time will return response code `409 Conflict` and explaining error message.
+* `POST /api/v1/email/confirm?site=site-id&tkn=token` - uses provided token parameter to set email for the user, _auth required_
+
+  Setting email subscribe user for all first-level replies to his messages.
+* `DELETE /api/v1/email?site=siteID` - removes user's email, _auth required_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -631,10 +830,17 @@ _returned id should be appended to load image url on caller side_
       Until     time.Time `json:"time"`
   }
   ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
-* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
+* `GET /api/v1/admin/export?site=site-id&mode=[stream|file]` - export all comments to json stream or gz file.
+* `POST /api/v1/admin/import?site=site-id` - import comments from the backup, uses post body.
+* `POST /api/v1/admin/import/form?site=site-id` - import comments from the backup, user post form.
+* `POST /api/v1/admin/remap?site=site-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
+From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
+export/import chain so make backup first.
+    ```
+    http://oldsite.com* https://newsite.com*
+    http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
+    ```
+* `GET /api/v1/admin/wait?site=site-id` - wait for completion for any async migration ops (import or remap).
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
 * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
 * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
@@ -673,5 +879,6 @@ _all admin calls require auth and admin privilege_
 * User can edit comments in 5 mins (configurable) window after creation.
 * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
 * All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
-* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
+* Images can be proxied (`IMAGE_PROXY_HTTP2HTTPS=true`) to prevent mixed http/https.
+* All images can be proxied and saved (`IMAGE_PROXY_CACHE_EXTERNAL=true`) instead of serving from original location. Beware, images which are posted with this parameter enabled will be served from proxy even after it will be disabled.
 * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.

backend/.golangci.yml

@@ -0,0 +1,71 @@
+run:
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0.1
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+
+linters:
+  enable:
+    - megacheck
+    - golint
+    - govet
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+    - stylecheck
+    - gochecknoinits
+    - scopelint
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
+  disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "should have a package comment, unless it's in another file for this package"
+      linters:
+        - golint
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false
+
+service:
+  golangci-lint-version: 1.23.x

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,22 @@
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+#ADD . /build/memory_store
+#WORKDIR /build/memory_store
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+
+FROM umputun/baseimage:app-latest
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,13 @@
+# sample store implementation 
+
+`memory_store` illustrates how to make a custom storage plugin for remark42. 
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. `docker-compose -f compose-dev-memstore.yml build`
+1. `docker-compose -f compose-dev-memstore.yml up`
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
+In real-life usage `replace github.com/umputun/remark/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key() (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, errors.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", errors.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, errors.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return errors.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated++ // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,562 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.Lock()
+	defer m.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", errors.New("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.Lock()
+	defer m.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, errors.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, errors.New("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.Lock()
+	defer m.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		m.Lock()
+		defer m.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.Lock()
+			defer m.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.Lock()
+	defer m.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return errors.New("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return errors.New("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	return status, errors.Wrapf(err, "failed to set flag %+v", req)
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, errors.New("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID == comment.ID && c.Locator == comment.Locator {
+			c.Text = comment.Text
+			c.Orig = comment.Orig
+			c.Score = comment.Score
+			c.Votes = comment.Votes
+			c.Pin = comment.Pin
+			c.Deleted = comment.Deleted
+			c.User = comment.User
+			comments[i] = c
+			m.posts[comment.Locator.SiteID] = comments
+			return nil
+		}
+	}
+	return errors.New("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,785 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds = make([]string, len(blockedList))
+	for i, x := range blockedList {
+		blockedIds[i] = x.ID
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		emailSet   = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.Unlock()
+
+	return nil
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.RUnlock()
+	if !ok {
+		return nil, errors.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.RLock()
+	img, ok := m.imagesStaging[id]
+	m.RUnlock()
+	if !ok {
+		return errors.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.Lock()
+	m.images[id] = img
+	m.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.RUnlock()
+
+	m.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,65 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: '2'
+
+services:
+
+  remark42:
+    build:
+      context: ../../..
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42-dev"
+    hostname: "remark42-dev"
+    restart: always
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - BACKUP_PATH=/srv/var/backup
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: ../../..
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,14 @@
+module github.com/umputun/remark/memory_store
+
+go 1.14
+
+require (
+	github.com/go-pkgz/jrpc v0.1.0
+	github.com/go-pkgz/lgr v0.7.0
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.5.1
+	github.com/umputun/go-flags v1.5.1
+	github.com/umputun/remark/backend v1.5.0
+)
+
+replace github.com/umputun/remark/backend => ../../

backend/_example/memory_store/go.sum

@@ -0,0 +1,269 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
+github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.6.0 h1:gcvXlpe0/NoQP3BvneRfgcauLIJDw9VblkoFwZ5XGFs=
+github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
+github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
+github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
+github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/chi v4.1.0+incompatible h1:ETj3cggsVIY2Xao5ExCu6YhEh5MD6JTfcBzS37R260w=
+github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.10.1/go.mod h1:wxyQqc0UUP1jT4l6zk1r6XPcVdcgIzW2OiQ8hBEHd64=
+github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
+github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
+github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
+github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
+github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
+github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
+github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
+github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
+github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/_example/memory_store/main.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/umputun/go-flags"
+
+	"github.com/umputun/remark/memory_store/accessor"
+	"github.com/umputun/remark/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
+
+	rpcServer := jrpc.Server{
+		API:        opts.API,
+		AuthUser:   opts.AuthUser,
+		AuthPasswd: opts.AuthPasswd,
+		Version:    revision,
+		AppName:    "remark42-memory",
+		Logger:     log.Default(),
+	}
+
+	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID:  "remark",
+		Enabled: true,
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	key, err := s.adm.Key()
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,306 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/server/rpc.go

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
+)
+
+// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
+// Note: this file can be used as-is in any custom jrpc plugin
+type RPC struct {
+	*jrpc.Server
+	eng engine.Interface
+	adm admin.Store
+	img image.Store
+}
+
+// NewRPC makes RPC instance and register handlers
+func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, img: i, Server: r}
+	res.addHandlers()
+	return res
+}
+
+func (s *RPC) addHandlers() {
+	// data store handlers
+	s.Group("store", jrpc.HandlersGroup{
+		"create":      s.createHndl,
+		"find":        s.findHndl,
+		"get":         s.getHndl,
+		"update":      s.updateHndl,
+		"count":       s.countHndl,
+		"info":        s.infoHndl,
+		"flag":        s.flagHndl,
+		"list_flags":  s.listFlagsHndl,
+		"user_detail": s.userDetailHndl,
+		"delete":      s.deleteHndl,
+		"close":       s.closeHndl,
+	})
+
+	// admin store handlers
+	s.Group("admin", jrpc.HandlersGroup{
+		"key":     s.admKeyHndl,
+		"admins":  s.admAdminsHndl,
+		"email":   s.admEmailHndl,
+		"enabled": s.admEnabledHndl,
+		"event":   s.admEventHndl,
+	})
+
+	// image store handlers
+	s.Group("image", jrpc.HandlersGroup{
+		"save_with_id": s.imgSaveWithIDHndl,
+		"load":         s.imgLoadHndl,
+		"commit":       s.imgCommitHndl,
+		"cleanup":      s.imgCleanupHndl,
+		"info":         s.imgInfoHndl,
+	})
+}

backend/_example/memory_store/server/rpc_test.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/memory_store/accessor"
+)
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func prepTestStore(t *testing.T) (port int, teardown func()) {
+	mg := accessor.NewMemData()
+	adm := accessor.NewMemAdminStore("secret")
+	img := accessor.NewMemImageStore()
+	s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+
+	admRec := accessor.AdminRec{
+		SiteID:  "test-site",
+		IDs:     []string{"id1", "id2"},
+		Email:   "admin@example.com",
+		Enabled: true,
+	}
+	adm.Set("test-site", admRec)
+
+	admRecDisabled := admRec
+	admRecDisabled.Enabled = false
+	adm.Set("test-site-disabled", admRecDisabled)
+
+	port = chooseRandomUnusedPort()
+	go func() {
+		_ = s.Run(port)
+	}()
+
+	waitForHTTPServerStart(port)
+
+	return port, func() {
+		require.NoError(t, s.Shutdown())
+	}
+}

backend/app/cmd/avatar.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"path"
-	"time"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -37,7 +34,7 @@ func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 }
 
 // Execute runs  with AvatarCommand parameters, entry point for "avatar" command
-func (ac *AvatarCommand) Execute(args []string) error {
+func (ac *AvatarCommand) Execute(_ []string) error {
 	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
 
 	src, err := ac.makeAvatarStore(ac.AvatarSrc)
@@ -78,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
 			return nil, err
@@ -93,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
 }
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}

backend/app/cmd/avatar_test.go

@@ -6,39 +6,22 @@ import (
 	"testing"
 
 	"github.com/go-pkgz/auth/avatar"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 )
 
 func TestAvatar_Execute(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -46,9 +29,9 @@ func TestAvatar_Execute(t *testing.T) {
 	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test2.db"})
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }
@@ -59,7 +42,7 @@ type avatarMigratorMock struct {
 	retCount int
 }
 
-func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
+func (a *avatarMigratorMock) Migrate(_, _ avatar.Store) (int, error) {
 	a.called++
 	return a.retCount, a.retError
 }

backend/app/cmd/backup.go

@@ -24,7 +24,7 @@ type BackupCommand struct {
 }
 
 // Execute runs export with ExportCommand parameters, entry point for "export" command
-func (ec *BackupCommand) Execute(args []string) error {
+func (ec *BackupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 

backend/app/cmd/backup_test.go

@@ -8,7 +8,8 @@ import (
 	"os"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,13 +26,13 @@ func TestBackup_Execute(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	defer os.Remove("/tmp/remark-test.export")
 
 	data, err := ioutil.ReadFile("/tmp/remark-test.export")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
@@ -49,7 +50,7 @@ func TestBackup_ExecuteFailedStatus(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
 }
@@ -68,7 +69,7 @@ func TestBackup_ExecuteFailedWrite(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
 		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
 }

backend/app/cmd/cleanup.go

@@ -34,7 +34,7 @@ var (
 
 // Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
 // This command uses provided flags to detect and remove junk comments
-func (cc *CleanupCommand) Execute(args []string) error {
+func (cc *CleanupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] cleanup for site %s", cc.Site)
 
 	posts, err := cc.postsInRange(cc.From, cc.To)
@@ -131,7 +131,8 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL) //nolint
+	client := http.Client{Timeout: 30 * time.Second}
+	r, err := client.Get(listURL)
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +159,8 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL) //nolint
+		client := http.Client{Timeout: 30 * time.Second}
+		r, err = client.Get(commentsURL)
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}

backend/app/cmd/cleanup_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi"
-	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -46,6 +46,7 @@ func TestCleanup_IsSpam(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
 		t.Run(checkName, func(t *testing.T) {
 			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
@@ -68,7 +69,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	posts, err := cmd.postsInRange("20181218", "20181219")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts))
@@ -78,7 +79,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	assert.Equal(t, 3, len(posts))
 
 	_, err = cmd.postsInRange("xxx", "yyy")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestCleanup_listComments(t *testing.T) {
@@ -91,7 +92,7 @@ func TestCleanup_listComments(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	comments, err := cmd.listComments("http://test.com/post1")
 	assert.NoError(t, err)
@@ -118,7 +119,7 @@ func TestCleanup_ExecuteSpam(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
 		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("deleted %+v", cleaned.ids)
@@ -136,7 +137,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("set titles for %+v", titledComments.ids)
@@ -144,7 +145,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 }
 
 func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
-	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
 		list := []store.PostInfo{
@@ -165,9 +166,9 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 			},
 		}
 		require.NoError(t, json.NewEncoder(w).Encode(list))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/find", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "remark", r.URL.Query().Get("site"))
 		require.Equal(t, "plain", r.URL.Query().Get("format"))
@@ -193,22 +194,22 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		}
 
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
 }

backend/app/cmd/cmd.go

@@ -21,6 +21,7 @@ import (
 type CommonOptionsCommander interface {
 	SetCommon(commonOpts CommonOpts)
 	Execute(args []string) error
+	HandleDeprecatedFlags() []DeprecatedFlag
 }
 
 // CommonOpts sets externally from main, shared across all commands
@@ -30,6 +31,13 @@ type CommonOpts struct {
 	Revision     string
 }
 
+// DeprecatedFlag contains information about deprecated option
+type DeprecatedFlag struct {
+	Old           string
+	New           string
+	RemoveVersion string
+}
+
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
@@ -38,6 +46,9 @@ func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
 	c.Revision = commonOpts.Revision
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated and returns their list
+func (c *CommonOpts) HandleDeprecatedFlags() []DeprecatedFlag { return nil }
+
 // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
 type fileParser struct {
 	site string

backend/app/cmd/cmd_test.go

@@ -31,7 +31,7 @@ func TestExport_ParseFileName(t *testing.T) {
 	for i, tt := range tbl {
 		r, err := tt.p.parse(now)
 		if tt.err {
-			assert.NotNil(t, err)
+			assert.Error(t, err)
 			continue
 		}
 		assert.Equal(t, tt.res, r, "check #%d", i)

backend/app/cmd/import.go

@@ -18,7 +18,7 @@ import (
 // ImportCommand set of flags and command for import
 type ImportCommand struct {
 	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
+	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint
 	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
 	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
 	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
@@ -26,7 +26,7 @@ type ImportCommand struct {
 }
 
 // Execute runs import with ImportCommand parameters, entry point for "import" command
-func (ic *ImportCommand) Execute(args []string) error {
+func (ic *ImportCommand) Execute(_ []string) error {
 	log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 
@@ -69,7 +69,7 @@ func (ic *ImportCommand) Execute(args []string) error {
 
 // reader returns reader for file. For .gz file wraps with gunzip
 func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
-	inpFile, err := os.Open(inp)
+	inpFile, err := os.Open(inp) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
 	}

backend/app/cmd/import_test.go

@@ -5,12 +5,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -21,7 +21,7 @@ func TestImport_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -34,7 +34,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -43,7 +43,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
@@ -61,21 +61,21 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on no such file")
-	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
+	assert.Error(t, err, "fail on no such file")
+	assert.Contains(t, err.Error(), "no such file or directory")
 
 	cmd = ImportCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on connection refused")
-	assert.True(t, strings.Contains(err.Error(), "connection refused"))
+	assert.Error(t, err, "fail on connection refused")
+	assert.Contains(t, err.Error(), "connection refused")
 
 	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		log.Printf("%+v", r)
@@ -87,10 +87,10 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestImport_ExecuteTimeout(t *testing.T) {
@@ -98,7 +98,7 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 		time.Sleep(500 * time.Millisecond)
 		fmt.Fprintln(w, "some response")
@@ -112,8 +112,8 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "deadline exceeded")
 }

backend/app/cmd/remap.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// RemapCommand set of flags and command for change linkage between comments to
+// different urls based on given rules (input file)
+type RemapCommand struct {
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
+	CommonOpts
+}
+
+// Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
+func (rc *RemapCommand) Execute(_ []string) error {
+	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	rulesReader, err := os.Open(rc.InputFile)
+	if err != nil {
+		return errors.Wrapf(err, "cant open file %s", rc.InputFile)
+	}
+
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
+	defer cancel()
+	remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
+	req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
+	if err != nil {
+		return errors.Wrapf(err, "can't make remap request for %s", remapURL)
+	}
+	req.SetBasicAuth("admin", rc.AdminPasswd)
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", remapURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "can't get response")
+	}
+
+	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+	return nil
+}

backend/app/cmd/remap_test.go

@@ -0,0 +1,37 @@
+package cmd
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/umputun/go-flags"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemap_Execute(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		body, err := ioutil.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(202)
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}

backend/app/cmd/restore_test.go

@@ -7,7 +7,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -19,7 +20,7 @@ func TestRestore_Execute(t *testing.T) {
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "native", r.URL.Query().Get("provider"))
 		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -32,7 +33,7 @@ func TestRestore_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }

backend/app/cmd/server.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -13,17 +14,20 @@ import (
 	"syscall"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
+	"github.com/dgrijalva/jwt-go"
+	"github.com/go-pkgz/jrpc"
 	log "github.com/go-pkgz/lgr"
+	"github.com/kyokomi/emoji"
 	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/provider"
+	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/mongo"
-	"github.com/go-pkgz/rest/cache"
+	cache "github.com/go-pkgz/lcw"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
@@ -38,31 +42,38 @@ import (
 
 // ServerCommand with command line flags and env
 type ServerCommand struct {
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
-	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
-	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
-	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
-	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Store      StoreGroup      `group:"store" namespace:"store" env-namespace:"STORE"`
+	Avatar     AvatarGroup     `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
+	Cache      CacheGroup      `group:"cache" namespace:"cache" env-namespace:"CACHE"`
+	Admin      AdminGroup      `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
+	Notify     NotifyGroup     `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	SMTP       SMTPGroup       `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
+	Image      ImageGroup      `group:"image" namespace:"image" env-namespace:"IMAGE"`
+	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Stream     StreamGroup     `group:"stream" namespace:"stream" env-namespace:"STREAM"`
+	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation  string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles  int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy      bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize  int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView       bool          `long:"simpler-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
 
 	Auth struct {
 		TTL struct {
@@ -73,13 +84,33 @@ type ServerCommand struct {
 		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
 		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
 		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
 		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
 		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Email     struct {
+			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
+			From         string        `long:"from" env:"FROM" description:"from email address"`
+			Subject      string        `long:"subj" env:"SUBJ" default:"remark42 confirmation" description:"email's subject"`
+			ContentType  string        `long:"content-type" env:"CONTENT_TYPE" default:"text/html" description:"content type"`
+			Host         string        `long:"host" env:"HOST" description:"[deprecated, use --smtp.host] SMTP host"`
+			Port         int           `long:"port" env:"PORT" description:"[deprecated, use --smtp.port] SMTP password"`
+			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"[deprecated, use --smtp.password] SMTP port"`
+			SMTPUserName string        `long:"user" env:"USER" description:"[deprecated, use --smtp.username] enable TLS"`
+			TLS          bool          `long:"tls" env:"TLS" description:"[deprecated, use --smtp.tls] SMTP TCP connection timeout"`
+			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"[deprecated, use --smtp.timeout] SMTP TCP connection timeout"`
+			MsgTemplate  string        `long:"template" env:"TEMPLATE" description:"message template file"`
+		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
 }
 
+// ImageProxyGroup defines options group for image proxy
+type ImageProxyGroup struct {
+	HTTP2HTTPS    bool `long:"http2https" env:"HTTP2HTTPS" description:"enable HTTP->HTTPS proxy"`
+	CacheExternal bool `long:"cache-external" env:"CACHE_EXTERNAL" description:"enable caching for external images"`
+}
+
 // AuthGroup defines options group for auth params
 type AuthGroup struct {
 	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
@@ -88,16 +119,17 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"` // nolint
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"rpc" default:"fs"` // nolint
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -106,26 +138,28 @@ type ImageGroup struct {
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
-	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
-	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+	MaxSize      int      `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int      `long:"resize-width" env:"RESIZE_WIDTH" default:"2400" description:"width of resized image"`
+	ResizeHeight int      `long:"resize-height" env:"RESIZE_HEIGHT" default:"900" description:"height of resized image"`
+	RPC          RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"uri" default:"fs"` //nolint
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
-	RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+	URI    string `long:"uri" env:"URI" default:"./var/avatars" description:"avatar's store URI"`
+	RszLmt int    `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
 }
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` // nolint
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -133,36 +167,46 @@ type CacheGroup struct {
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
-	URL string `long:"url" env:"URL" description:"mongo url"`
-	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"` //nolint
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
+}
+
+// SMTPGroup defines options for SMTP server connection, used in auth and notify modules
+type SMTPGroup struct {
+	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
+	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
+	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Password string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	TLS      bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	TimeOut  time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
 }
 
 // NotifyGroup defines options for notification
 type NotifyGroup struct {
-	Type      string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" default:"none"`
-	QueueSize int    `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
+	Type      []string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" choice:"email" default:"none" env-delim:","` //nolint
+	QueueSize int      `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
 	Telegram  struct {
 		Token   string        `long:"token" env:"TOKEN" description:"telegram token"`
 		Channel string        `long:"chan" env:"CHAN" description:"telegram channel"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"telegram timeout"`
 		API     string        `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
 	} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
+	Email struct {
+		From                string `long:"fromAddress" env:"FROM" description:"from email address"`
+		VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
+		AdminNotifications  bool   `long:"notify_admin" env:"ADMIN" description:"notify admin on new comments via ADMIN_SHARED_EMAIL"`
+	} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 }
 
 // SSLGroup defines options group for server ssl params
 type SSLGroup struct {
-	Type         string `long:"type" env:"TYPE" description:"ssl (auto)support" choice:"none" choice:"static" choice:"auto" default:"none"`
+	Type         string `long:"type" env:"TYPE" description:"ssl (auto) support" choice:"none" choice:"static" choice:"auto" default:"none"` //nolint
 	Port         int    `long:"port" env:"PORT" description:"port number for https server" default:"8443"`
 	Cert         string `long:"cert" env:"CERT" description:"path to cert.pem file"`
 	Key          string `long:"key" env:"KEY" description:"path to key.pem file"`
@@ -170,6 +214,27 @@ type SSLGroup struct {
 	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
 }
 
+// StreamGroup define options for streaming apis
+type StreamGroup struct {
+	RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
+	TimeOut         time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
+	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
+}
+
+// RPCGroup defines options for remote modules (plugins)
+type RPCGroup struct {
+	API          string        `long:"api" env:"API" description:"rpc extension api url"`
+	TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+	AuthUser     string        `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req cache.FlusherRequest)                                        // evict matched records
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -181,11 +246,12 @@ type serverApp struct {
 	avatarStore   avatar.Store
 	notifyService *notify.Service
 	imageService  *image.Service
+	authenticator *auth.Service
 	terminated    chan struct{}
 }
 
 // Execute is the entry point for "server" command, called by flag parser
-func (s *ServerCommand) Execute(args []string) error {
+func (s *ServerCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start server on port %d", s.Port)
 	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC", "ADMIN_PASSWD")
 
@@ -211,6 +277,40 @@ func (s *ServerCommand) Execute(args []string) error {
 	return nil
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated returns their list
+func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
+	// 1.5.0
+	if s.Auth.Email.Host != "" && s.SMTP.Host == "" {
+		s.SMTP.Host = s.Auth.Email.Host
+		result = append(result, DeprecatedFlag{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.Port != 0 && s.SMTP.Port == 0 {
+		s.SMTP.Port = s.Auth.Email.Port
+		result = append(result, DeprecatedFlag{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TLS && !s.SMTP.TLS {
+		s.SMTP.TLS = s.Auth.Email.TLS
+		result = append(result, DeprecatedFlag{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPUserName != "" && s.SMTP.Username == "" {
+		s.SMTP.Username = s.Auth.Email.SMTPUserName
+		result = append(result, DeprecatedFlag{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.SMTPPassword != "" && s.SMTP.Password == "" {
+		s.SMTP.Password = s.Auth.Email.SMTPPassword
+		result = append(result, DeprecatedFlag{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"})
+	}
+	if s.Auth.Email.TimeOut != 10*time.Second && s.SMTP.TimeOut == 10*time.Second {
+		s.SMTP.TimeOut = s.Auth.Email.TimeOut
+		result = append(result, DeprecatedFlag{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"})
+	}
+	if s.LegacyImageProxy && !s.ImageProxy.HTTP2HTTPS {
+		s.ImageProxy.HTTP2HTTPS = s.LegacyImageProxy
+		result = append(result, DeprecatedFlag{Old: "img-proxy", New: "image-proxy.http2https", RemoveVersion: "1.7.0"})
+	}
+	return result
+}
+
 // newServerApp prepares application and return it with all active parts
 // doesn't start anything
 func (s *ServerCommand) newServerApp() (*serverApp, error) {
@@ -238,9 +338,10 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make pictures store")
 	}
+	log.Printf("[DEBUG] image service for url=%s, EditDuration=%v", imageService.ImageAPI, imageService.EditDuration)
 
 	dataService := &service.DataStore{
-		Interface:              storeEngine,
+		Engine:                 storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
@@ -250,6 +351,8 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
+	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
+	dataService.RestrictSameIPVotes.Duration = s.DurationVoteIP
 
 	loadingCache, err := s.makeCache()
 	if err != nil {
@@ -270,17 +373,38 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
 		WordPressImporter: &migrator.WordPress{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
+		URLMapperMaker:    migrator.NewURLMapper,
 		KeyStore:          adminStore,
 	}
 
-	notifyService, err := s.makeNotify(dataService)
+	var emailNotifications bool
+	notifyService, err := s.makeNotify(dataService, authenticator)
+
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "email":
+			emailNotifications = true
+		}
+	}
+
 	if err != nil {
 		log.Printf("[WARN] failed to make notify service, %s", err)
 		notifyService = notify.NopService // disable notifier
+		emailNotifications = false        // email notifications are not available in this case
 	}
 
-	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
-	commentFormatter := store.NewCommentFormatter(imgProxy)
+	imgProxy := &proxy.Image{
+		HTTP2HTTPS:    s.ImageProxy.HTTP2HTTPS,
+		CacheExternal: s.ImageProxy.CacheExternal,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     s.RemarkURL,
+		ImageService:  imageService,
+	}
+	emojiFmt := store.CommentConverterFunc(func(text string) string { return text })
+	if s.EnableEmoji {
+		emojiFmt = func(text string) string { return emoji.Sprint(text) }
+	}
+	commentFormatter := store.NewCommentFormatter(imgProxy, emojiFmt)
 
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
@@ -303,6 +427,20 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
 		ImageService:     imageService,
+		Streamer: &api.Streamer{
+			TimeOut:   s.Stream.TimeOut,
+			Refresh:   s.Stream.RefreshInterval,
+			MaxActive: int32(s.Stream.MaxActive),
+		},
+		EmailNotifications: emailNotifications,
+		EmojiEnabled:       s.EnableEmoji,
+		AnonVote:           s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:         s.SimpleView,
+	}
+
+	// enable admin notifications only if admin email is set
+	if s.Notify.Email.AdminNotifications && s.Admin.Shared.Email != "" {
+		srv.AdminEmail = s.Admin.Shared.Email
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -326,6 +464,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
 		imageService:  imageService,
+		authenticator: authenticator,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -341,18 +480,6 @@ func (a *serverApp) run(ctx context.Context) error {
 		<-ctx.Done()
 		log.Print("[INFO] shutdown initiated")
 		a.restSrv.Shutdown()
-		if a.devAuth != nil {
-			a.devAuth.Shutdown()
-		}
-		if e := a.dataService.Close(); e != nil {
-			log.Printf("[WARN] failed to close data store, %s", e)
-		}
-		if e := a.avatarStore.Close(); e != nil {
-			log.Printf("[WARN] failed to close avatar store, %s", e)
-		}
-		a.notifyService.Close()
-		a.imageService.Close()
-		log.Print("[INFO] shutdown completed")
 	}()
 
 	a.activateBackup(ctx) // runs in goroutine for each site
@@ -360,9 +487,31 @@ func (a *serverApp) run(ctx context.Context) error {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
 
+	// staging images resubmit after restart of the app
+	if e := a.dataService.ResubmitStagingImages(a.Sites); e != nil {
+		log.Printf("[WARN] failed to resubmit comments with staging images, %s", e)
+	}
+
 	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
 
 	a.restSrv.Run(a.Port)
+
+	// shutdown procedures after HTTP server is stopped
+	if a.devAuth != nil {
+		a.devAuth.Shutdown()
+	}
+	if e := a.dataService.Close(); e != nil {
+		log.Printf("[WARN] failed to close data store, %s", e)
+	}
+	if e := a.avatarStore.Close(); e != nil {
+		log.Printf("[WARN] failed to close avatar store, %s", e)
+	}
+	a.notifyService.Close()
+	// call potentially infinite loop with cancellation after a minute as a safeguard
+	minuteCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	a.imageService.Close(minuteCtx)
+
 	close(a.terminated)
 	return nil
 }
@@ -400,13 +549,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return result, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+	case "rpc":
+		r := &engine.RPC{Client: jrpc.Client{
+			API:        s.Store.RPC.API,
+			Client:     http.Client{Timeout: s.Store.RPC.TimeOut},
+			AuthUser:   s.Store.RPC.AuthUser,
+			AuthPasswd: s.Store.RPC.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
 	}
@@ -422,39 +572,50 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
-	case "mongo":
-		mgServer, err := s.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
 			return nil, err
 		}
 		return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
+	case "uri":
+		return avatar.NewStore(s.Avatar.URI)
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	imageServiceParams := image.ServiceParams{
+		ImageAPI:     s.RemarkURL + "/api/v1/picture/",
+		ProxyAPI:     s.RemarkURL + "/api/v1/img",
+		EditDuration: s.EditDuration,
+		MaxSize:      s.Image.MaxSize,
+		MaxHeight:    s.Image.ResizeHeight,
+		MaxWidth:     s.Image.ResizeWidth,
+	}
 	switch s.Image.Type {
+	case "bolt":
+		boltImageStore, err := image.NewBoltStorage(s.Image.Bolt.File, bolt.Options{})
+		if err != nil {
+			return nil, err
+		}
+		return image.NewService(boltImageStore, imageServiceParams), nil
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
 		}
-		return &image.Service{
-			Store: &image.FileSystem{
-				Location:   s.Image.FS.Path,
-				Staging:    s.Image.FS.Staging,
-				Partitions: s.Image.FS.Partitions,
-				MaxSize:    s.Image.MaxSize,
-				MaxHeight:  s.Image.ResizeHeight,
-				MaxWidth:   s.Image.ResizeWidth,
-			},
-			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
-		}, nil
+		return image.NewService(&image.FileSystem{
+			Location:   s.Image.FS.Path,
+			Staging:    s.Image.FS.Staging,
+			Partitions: s.Image.FS.Partitions,
+		}, imageServiceParams), nil
+	case "rpc":
+		return image.NewService(&image.RPC{
+			Client: jrpc.Client{
+				API:        s.Image.RPC.API,
+				Client:     http.Client{Timeout: s.Image.RPC.TimeOut},
+				AuthUser:   s.Image.RPC.AuthUser,
+				AuthPasswd: s.Image.RPC.AuthPassword,
+			}}, imageServiceParams), nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
 }
@@ -469,45 +630,57 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 				s.Admin.Shared.Email = "admin@" + u.Host
 			}
 		}
-		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return nil, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
-		return admin.NewMongoStore(conn, s.SharedSecret), nil
+		return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
+	case "rpc":
+		r := &admin.RPC{Client: jrpc.Client{
+			API:        s.Admin.RPC.API,
+			Client:     http.Client{Timeout: s.Admin.RPC.TimeOut},
+			AuthUser:   s.Admin.RPC.AuthUser,
+			AuthPasswd: s.Admin.RPC.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
 	}
 }
 
-func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
+func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
 	switch s.Cache.Type {
 	case "mem":
-		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
-	// case "mongo":
-	// 	mgServer, err := s.makeMongo()
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "failed to create mongo server")
-	// 	}
-	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
-	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-	// 		cache.MaxKeys(s.Cache.Max.Items))
+		if err != nil {
+			return nil, errors.Wrap(err, "cache backend initialization")
+		}
+		return cache.NewScache(backend), nil
 	case "none":
-		return &cache.Nop{}, nil
+		return cache.NewScache(&cache.Nop{}), nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
-	if s.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
-}
+var msgTemplate = `
+<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
+	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
+		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and paste this text into “token” field on comments page</i></p>
+		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+	</div>
+	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
+</div>
+</body>
+</html>
+`
 
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
@@ -528,12 +701,33 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
 		providers++
 	}
+	if s.Auth.Twitter.CID != "" && s.Auth.Twitter.CSEC != "" {
+		authenticator.AddProvider("twitter", s.Auth.Twitter.CID, s.Auth.Twitter.CSEC)
+		providers++
+	}
+
 	if s.Auth.Dev {
 		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Email.Enable {
+		params := sender.EmailParams{
+			Host:         s.SMTP.Host,
+			Port:         s.SMTP.Port,
+			SMTPUserName: s.SMTP.Username,
+			SMTPPassword: s.SMTP.Password,
+			TimeOut:      s.SMTP.TimeOut,
+			TLS:          s.SMTP.TLS,
+			From:         s.Auth.Email.From,
+			Subject:      s.Auth.Email.Subject,
+			ContentType:  s.Auth.Email.ContentType,
+		}
+		sndr := sender.NewEmailClient(params, log.Default())
+		authenticator.AddVerifProvider("email", s.loadEmailTemplate(), sndr)
+	}
+
 	if s.Auth.Anonymous {
 		log.Print("[INFO] anonymous access enabled")
 		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
@@ -557,20 +751,83 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 	}
 }
 
-func (s *ServerCommand) makeNotify(dataStore *service.DataStore) (*notify.Service, error) {
-	log.Printf("[INFO] make notify, type=%s", s.Notify.Type)
-	switch s.Notify.Type {
-	case "telegram":
-		tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
-			s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create telegram notification destination")
+// loadEmailTemplate trying to get template from opts MsgTemplate and default to embedded
+// if not defined or failed to load
+func (s *ServerCommand) loadEmailTemplate() string {
+	tmpl := msgTemplate
+	if s.Auth.Email.MsgTemplate != "" {
+		log.Printf("[DEBUG] load email template from %s", s.Auth.Email.MsgTemplate)
+		b, err := ioutil.ReadFile(s.Auth.Email.MsgTemplate)
+		if err == nil {
+			tmpl = string(b)
+		} else {
+			log.Printf("[WARN] failed to load email template from %s, %v", s.Auth.Email.MsgTemplate, err)
 		}
-		return notify.NewService(dataStore, s.Notify.QueueSize, tg), nil
-	case "none":
-		return notify.NopService, nil
 	}
-	return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+	return tmpl
+}
+
+func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *auth.Service) (*notify.Service, error) {
+	var notifyService *notify.Service
+	var destinations []notify.Destination
+	for _, t := range s.Notify.Type {
+		switch t {
+		case "telegram":
+			tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
+				s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create telegram notification destination")
+			}
+			destinations = append(destinations, tg)
+		case "email":
+			emailParams := notify.EmailParams{
+				From:                s.Notify.Email.From,
+				VerificationSubject: s.Notify.Email.VerificationSubject,
+				UnsubscribeURL:      s.RemarkURL + "/email/unsubscribe.html",
+				// TODO: uncomment after #560 frontend part is ready and URL is known
+				// SubscribeURL:        s.RemarkURL + "/subscribe.html?token=",
+				TokenGenFn: func(userID, email, site string) (string, error) {
+					claims := token.Claims{
+						Handshake: &token.Handshake{ID: userID + "::" + email},
+						StandardClaims: jwt.StandardClaims{
+							Audience:  site,
+							ExpiresAt: time.Now().Add(100 * 365 * 24 * time.Hour).Unix(),
+							NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+							Issuer:    "remark42",
+						},
+					}
+					tkn, err := authenticator.TokenService().Token(claims)
+					if err != nil {
+						return "", errors.Wrapf(err, "failed to make unsubscription token")
+					}
+					return tkn, nil
+				},
+			}
+			smtpParams := notify.SMTPParams{
+				Host:     s.SMTP.Host,
+				Port:     s.SMTP.Port,
+				TLS:      s.SMTP.TLS,
+				Username: s.SMTP.Username,
+				Password: s.SMTP.Password,
+				TimeOut:  s.SMTP.TimeOut,
+			}
+			emailService, err := notify.NewEmail(emailParams, smtpParams)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create email notification destination")
+			}
+			destinations = append(destinations, emailService)
+		case "none":
+			notifyService = notify.NopService
+		default:
+			return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+		}
+	}
+
+	if len(destinations) != 0 {
+		log.Printf("[INFO] make notify, types=%s", s.Notify.Type)
+		notifyService = notify.NewService(dataStore, s.Notify.QueueSize, destinations...)
+	}
+	return notifyService, nil
 }
 
 func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
@@ -610,7 +867,7 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 		TokenDuration:  s.Auth.TTL.JWT,
 		CookieDuration: s.Auth.TTL.Cookie,
 		SecureCookies:  strings.HasPrefix(s.RemarkURL, "https://"),
-		SecretReader: token.SecretFunc(func() (string, error) { // get secret per site
+		SecretReader: token.SecretFunc(func(aud string) (string, error) { // get secret per site
 			return admns.Key()
 		}),
 		ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
@@ -619,6 +876,26 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			c.User.SetAdmin(ds.IsAdmin(c.Audience, c.User.ID))
 			c.User.SetBoolAttr("blocked", ds.IsBlocked(c.Audience, c.User.ID))
+			var err error
+			c.User.Email, err = ds.GetUserEmail(c.Audience, c.User.ID)
+			if err != nil {
+				log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
+			}
+
+			// don't allow anonymous with admin's name
+			if strings.HasPrefix(c.User.ID, "anonymous_") {
+				admins, err := admns.Admins(c.Audience)
+				if err != nil {
+					log.Printf("[WARN] can't get admins for %s, %v", c.Audience, err)
+				}
+				for _, a := range admins {
+					if strings.EqualFold(c.User.Name, a) {
+						c.User.SetBoolAttr("blocked", true)
+						break
+					}
+				}
+			}
+
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
@@ -626,13 +903,18 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			if claims.User == nil {
 				return false
 			}
+			if claims.User.Audience == "" { // reject empty aud, made with old (pre 0.8.x) version of auth package
+				return false
+			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
 		Logger:            log.Default(),
 		RefreshCache:      newAuthRefreshCache(),
+		UseGravatar:       true,
 	})
 	s.addAuthProviders(authenticator)
 	return authenticator

backend/app/cmd/server_test.go

@@ -5,206 +5,204 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/globalsign/mgo"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
-	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 	body, _ = ioutil.ReadAll(resp.Body)
 	t.Log(string(body))
 
-	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+	email, err := app.dataService.AdminStore.Email("")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_AnonMode(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		o.Auth.Anonymous = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
-	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+	require.Equal(t, 5+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[5].Name(), "anon auth provider")
 
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as good anonymous
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims := getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 
 	// try to login with bad name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
-	require.Nil(t, err)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 403, resp.StatusCode)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
+	// try to login with admin name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umputun&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as anonymous with admin name
+	client = http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims = getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	assert.True(t, claims.User.BoolAttr("blocked"), "should be blocked")
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+
+	cancel()
 	app.Wait()
 }
-func TestServerApp_WithMongo(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
+func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (token string, claims token.Claims) {
+	var err error
+	for _, c := range resp.Cookies() {
+		if c.Name == "JWT" {
+			token = c.Value
+			claims, err = app.restSrv.Authenticator.TokenService().Parse(c.Value)
+			require.NoError(t, err)
+		}
 	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
-
-	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
-	// prepare options
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
-		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
-	require.Nil(t, err)
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
-
-	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
-
-	defer func() {
-		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, e)
-		conn := mongo.NewConnection(s, "test_remark", "")
-		_ = conn.WithDB(func(dbase *mgo.Database) error {
-			assert.NoError(t, dbase.DropDatabase())
-			return nil
-		})
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
-
-	app.Wait()
+	return token, claims
 }
 
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://localhost:18443", SharedSecret: "123456"})
+	sslPort := chooseRandomUnusedPort()
+	opts.SetCommon(CommonOpts{RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort), SharedSecret: "123456"})
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=" + strconv.Itoa(port), "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
 		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
 		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
-		"--ssl.port=18443", "--image.fs.path=/tmp"})
-	require.Nil(t, err)
+		"--ssl.port=" + strconv.Itoa(sslPort), "--image.fs.path=/tmp"})
+	require.NoError(t, err)
 
 	// create app
 	app, err := opts.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(1 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -219,21 +217,58 @@ func TestServerApp_WithSSL(t *testing.T) {
 	}
 
 	// check http to https redirect response
-	resp, err := client.Get("http://localhost:18080/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:18443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	// check https server
-	resp, err = client.Get("https://localhost:18443/ping")
-	require.Nil(t, err)
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
+	app.Wait()
+}
+
+func TestServerApp_WithRemote(t *testing.T) {
+
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
+		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+	require.NoError(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+	// create app
+	app, err := opts.newServerApp()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { _ = app.run(ctx) }()
+	waitForHTTPServerStart(port)
+
+	// send ping
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	cancel()
 	app.Wait()
 }
 
@@ -245,7 +280,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 	// RO bolt location
 	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -255,7 +290,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
 	t.Log(err)
@@ -265,7 +300,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
@@ -274,7 +309,7 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
-	assert.NotNil(t, err, "blah is invalid type")
+	assert.Error(t, err, "blah is invalid type")
 
 	opts.Store.Type = "blah"
 	_, err = opts.newServerApp()
@@ -283,38 +318,86 @@ func TestServerApp_Failed(t *testing.T) {
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = chooseRandomUnusedPort()
 		return o
 	})
+	time.AfterFunc(100*time.Millisecond, func() {
+		cancel()
+	})
 	st := time.Now()
 	err := app.run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err)
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100msec")
 	app.Wait()
 }
 
 func TestServerApp_MainSignal(t *testing.T) {
 
+	done := make(chan struct{})
 	go func() {
+		<-done
 		time.Sleep(250 * time.Millisecond)
 		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}()
-	st := time.Now()
 
 	s := ServerCommand{}
 	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	p := flags.NewParser(&s, flags.Default)
+	port := chooseRandomUnusedPort()
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=" + strconv.Itoa(port), "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	st := time.Now()
+	close(done)
 	err = s.Execute(args)
-	assert.NoError(t, err, "execute failed")
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err, "execute should be without errors")
+	assert.True(t, time.Since(st).Seconds() < 5, "should take under five sec", time.Since(st).Seconds())
+}
+
+func TestServerApp_DeprecatedArgs(t *testing.T) {
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{
+		"test",
+		"--auth.email.host=smtp.example.org",
+		"--auth.email.port=666",
+		"--auth.email.tls",
+		"--auth.email.user=test_user",
+		"--auth.email.passwd=test_password",
+		"--auth.email.timeout=15s",
+	}
+	assert.Empty(t, s.SMTP.Host)
+	assert.Empty(t, s.SMTP.Port)
+	assert.Empty(t, s.SMTP.TLS)
+	assert.Empty(t, s.SMTP.Username)
+	assert.Empty(t, s.SMTP.Password)
+	assert.Empty(t, s.SMTP.TimeOut)
+	_, err := p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlags := s.HandleDeprecatedFlags()
+	assert.ElementsMatch(t,
+		[]DeprecatedFlag{
+			{Old: "auth.email.host", New: "smtp.host", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.port", New: "smtp.port", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.tls", New: "smtp.tls", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"},
+			{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"},
+		},
+		deprecatedFlags)
+	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
+	assert.Equal(t, 666, s.SMTP.Port)
+	assert.Equal(t, true, s.SMTP.TLS)
+	assert.Equal(t, "test_user", s.SMTP.Username)
+	assert.Equal(t, "test_password", s.SMTP.Password)
+	assert.Equal(t, 15*time.Second, s.SMTP.TimeOut)
 }
 
 func Test_ACMEEmail(t *testing.T) {
@@ -323,9 +406,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	args := []string{"--ssl.type=auto"}
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err := cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -333,9 +416,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--ssl.acme-email=adminname@adminhost.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "adminname@adminhost.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -343,9 +426,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared", "--admin.shared.email=superadmin@admin.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "superadmin@admin.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -353,20 +436,21 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// make a token for user dev
 	tkService := app.restSrv.Authenticator.TokenService()
@@ -389,51 +473,122 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
+	// add comment with no-aud claim
+	claimsNoAud := claims
+	claimsNoAud.Audience = ""
+	tkNoAud, err := tkService.Token(claimsNoAud)
+	require.NoError(t, err)
+	t.Logf("no-aud claims: %s", tkNoAud)
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/", 
+"site": "remark"}}`))
+	require.NoError(t, err)
+	req.Header.Set("X-JWT", tkNoAud)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
+
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
-	assert.Nil(t, e)
+	req, err = http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, e = client.Do(req)
-	require.Nil(t, e)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Log(string(b))
 
-	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
-
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
+	body, err = ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
+		"blocked user can't post, \n"+tk+"\n"+string(body))
 
+	cancel()
 	app.Wait()
 }
 
-func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
+func TestServer_loadEmailTemplate(t *testing.T) {
+	cmd := ServerCommand{}
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
+	r := cmd.loadEmailTemplate()
+	assert.Equal(t, "The token is {{.Token}}", r)
+
+	cmd.Auth.Email.MsgTemplate = ""
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+
+	cmd.Auth.Email.MsgTemplate = "bad-file"
+	r = cmd.loadEmailTemplate()
+	assert.Contains(t, r, "Remark42</h1>")
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}
+
+func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
 	cmd := ServerCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "secret"})
 
 	// prepare options
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
@@ -441,23 +596,29 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
 	cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
+	cmd.Auth.Email.Enable = true
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
 	cmd.BackupLocation = "/tmp"
-	cmd.Notify.Type = "telegram"
-	cmd.Notify.Telegram.API = "http://127.0.0.1:12340/"
-	cmd.Notify.Telegram.Token = "blah"
+	cmd.Notify.Type = []string{"email"}
+	cmd.Notify.Email.From = "from@example.org"
+	cmd.Notify.Email.VerificationSubject = "test verification email subject"
+	cmd.SMTP.Host = "127.0.0.1"
+	cmd.SMTP.Port = 25
+	cmd.SMTP.Username = "test_user"
+	cmd.SMTP.Password = "test_password"
+	cmd.SMTP.TimeOut = time.Second
+	cmd.UpdateLimit = 10
+	cmd.Admin.Type = "shared"
+	cmd.Admin.Shared.Admins = []string{"umputun", "bobuk"}
 	cmd = fn(cmd)
 
 	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
 
 	// create app
 	app, err := cmd.newServerApp()
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	return app, ctx
+	rand.Seed(time.Now().UnixNano())
+	return app, ctx, cancel
 }

backend/app/cmd/testdata/email.tmpl

@@ -0,0 +1 @@
+The token is {{.Token}}

backend/app/cmd/testdata/remap_urls.txt

@@ -0,0 +1,2 @@
+http://oldsite.com* https://newsite.com*
+http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/umputun/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )
@@ -21,6 +21,7 @@ type Opts struct {
 	RestoreCmd cmd.RestoreCommand `command:"restore"`
 	AvatarCmd  cmd.AvatarCommand  `command:"avatar"`
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
 	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
@@ -44,6 +45,10 @@ func main() {
 			SharedSecret: opts.SharedSecret,
 			Revision:     revision,
 		})
+		for _, entry := range c.HandleDeprecatedFlags() {
+			log.Printf("[WARN] --%s is deprecated and will be removed in v%s, please use --%s instead",
+				entry.Old, entry.RemoveVersion, entry.New)
+		}
 		err := c.Execute(args)
 		if err != nil {
 			log.Printf("[ERROR] failed with %+v", err)
@@ -79,6 +84,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
+// nolint:gochecknoinits
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal)

backend/app/main_test.go

@@ -1,19 +1,18 @@
 package main
 
 import (
-	"context"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/repeater"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,45 +23,37 @@ func Test_Main(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
+	port := chooseRandomUnusedPort()
 	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
-		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+		"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
+	done := make(chan struct{})
 	go func() {
-		time.Sleep(5000 * time.Millisecond)
+		<-done
 		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, e)
+		require.NoError(t, e)
 	}()
 
-	wg := sync.WaitGroup{}
-	wg.Add(1)
+	finished := make(chan struct{})
 	go func() {
-		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
-		wg.Done()
+		close(finished)
 	}()
 
-	var passed bool
-	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
-		resp, e := http.Get("http://localhost:18222/api/v1/ping")
-		if e != nil {
-			t.Logf("%+v", e)
-			return e
-		}
-		require.Nil(t, e)
-		defer resp.Body.Close()
-		assert.Equal(t, 200, resp.StatusCode)
-		body, e := ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
-		assert.Equal(t, "pong", string(body))
-		passed = true
-		return nil
-	})
+	// defer cleanup because require check below can fail
+	defer func() {
+		close(done)
+		<-finished
+	}()
 
+	waitForHTTPServerStart(port)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, true, passed, "at least on ping passed")
-
-	wg.Wait()
+	assert.Equal(t, "pong", string(body))
 }
 
 func TestGetDump(t *testing.T) {
@@ -70,5 +61,28 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "goroutine"))
 	assert.True(t, strings.Contains(dump, "[running]"))
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
-	log.Printf("\n dump: %s", dump)
+	t.Logf("\n dump: %s", dump)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 10 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 100; i++ {
+		time.Sleep(time.Millisecond * 100)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
 }

backend/app/migrator/backup_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBackup_RemoveOldBackupFiles(t *testing.T) {
@@ -21,17 +22,17 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 	}
 	fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
 	err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
 	bk.removeOldBackupFiles()
 	ff, err := ioutil.ReadDir(loc)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
 	assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
 	assert.Equal(t, "backup-site1-20171209.gz", ff[1].Name())
 	assert.Equal(t, "backup-site1-20171210.gz", ff[2].Name())
@@ -76,7 +77,7 @@ func TestBackup_Do(t *testing.T) {
 
 type mockExporter struct{}
 
-func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
+func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
 	_, err := w.Write([]byte("some export blah blah 1234567890"))
 	return 1000, err
 }

backend/app/migrator/disqus_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -19,16 +19,17 @@ import (
 func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	d := Disqus{DataStore: &dataStore}
 	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(last), "4 comments imported")
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(last), "4 comments imported")
 
 	c := last[len(last)-1] // last reverses, get first one
 	assert.True(t, strings.HasPrefix(c.Text, "<p>The quick brown fox"))
@@ -40,11 +41,11 @@ func TestDisqus_Import(t *testing.T) {
 	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
 
 	posts, err := dataStore.List("test", 0, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts), "2 posts")
 
 	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, count)
 }
 
@@ -56,7 +57,7 @@ func TestDisqus_Convert(t *testing.T) {
 	for comment := range ch {
 		res = append(res, comment)
 	}
-	assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
+	require.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
 
 	exp0 := store.Comment{
 		ID: "299619020",

backend/app/migrator/mapper.go

@@ -0,0 +1,74 @@
+package migrator
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+// URLMapper implements Mapper interface
+type URLMapper struct {
+	rules map[string]string
+}
+
+// NewURLMapper reads rules from given reader and returns initialized URLMapper
+// if given rules are valid.
+func NewURLMapper(reader io.Reader) (Mapper, error) {
+	u := &URLMapper{}
+	if err := u.loadRules(reader); err != nil {
+		return u, err
+	}
+	return u, nil
+}
+
+// loadRules loads url-mapping rules from reader to mapper.
+// Rules must be a text consists of rows separated by \n.
+// Each row holds from-url and to-url separated by space.
+// If urls end with asterisk (*) it means try to match by prefix.
+// Example:
+// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
+// https://www.myblog.com/* https://myblog.com/*
+func (u *URLMapper) loadRules(reader io.Reader) error {
+	data, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	rulesText := strings.TrimSpace(string(data))
+
+	u.rules = make(map[string]string)
+
+	for _, row := range strings.Split(rulesText, "\n") {
+		row = strings.TrimSpace(row)
+		urls := strings.Split(row, " ")
+		if len(urls) != 2 {
+			return errors.New("bad row " + row)
+		}
+
+		from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
+		u.rules[from] = to
+	}
+	return nil
+}
+
+// URL maps given url to another url according loaded url-rules.
+// If not matched returns given url.
+func (u *URLMapper) URL(url string) string {
+	if newURL, ok := u.rules[url]; ok {
+		return newURL
+	}
+	// try to match by prefix
+	for oldURL, newURL := range u.rules {
+		if !strings.HasSuffix(oldURL, "*") {
+			continue
+		}
+		oldURL = strings.TrimSuffix(oldURL, "*")
+		newURL = strings.TrimSuffix(newURL, "*")
+		if strings.HasPrefix(url, oldURL) {
+			return newURL + strings.TrimPrefix(url, oldURL)
+		}
+	}
+	// search failed, return given url
+	return url
+}

backend/app/migrator/mapper_test.go

@@ -0,0 +1,90 @@
+package migrator
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUrlMapper_URL(t *testing.T) {
+	// want remap urls from https://radio-t.com to https://www.radio-t.com
+	// also map individual urls
+	rules := strings.NewReader(`
+https://radio-t.com* https://www.radio-t.com*
+https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
+https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
+`)
+
+	mapper, err := NewURLMapper(rules)
+	assert.NoError(t, err)
+
+	// if url not matched mapper should return given url
+	assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
+	assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
+	// check strict matching
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
+	// check pattern matching (by prefix)
+	assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
+
+	// want remap from http to https
+	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
+	mapper, err = NewURLMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
+
+	// want remap from http to https by pattern
+	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
+	mapper, err = NewURLMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+}
+
+func TestUrlMapper_New(t *testing.T) {
+	cases := []struct {
+		rules       string
+		expectError bool
+	}{
+		// bad input, expect error
+		{
+			rules:       "https://radio-t.com ",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com\n https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com   \n https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+
+		// valid input, no error
+		{
+			rules: "https://radio-t.com* https://www.radio-t.com*",
+		},
+		{
+			rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
+		},
+		{
+			rules: "https://any.com/p/111 https://any.com/p/222   \n https://any.com/p/333 https://any.com/p/222   \n",
+		},
+	}
+	for _, c := range cases {
+		_, err := NewURLMapper(strings.NewReader(c.rules))
+		if c.expectError {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}

backend/app/migrator/migrator.go

@@ -24,10 +24,20 @@ type Exporter interface {
 	Export(w io.Writer, siteID string) (int, error)
 }
 
+// Mapper defines interface to convert data in import procedure
+type Mapper interface {
+	URL(url string) string
+}
+
+// MapperMaker defines function that reads rules from reader and
+// returns new Mapper with loaded rules. If rules are not valid
+// it returns error.
+type MapperMaker func(reader io.Reader) (Mapper, error)
+
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
@@ -42,6 +52,8 @@ type ImportParams struct {
 	SiteID    string
 }
 
+var adminUser = store.User{Admin: true}
+
 // ImportComments imports from given provider format and saves to store
 func ImportComments(p ImportParams) (int, error) {
 	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)

backend/app/migrator/migrator_test.go

@@ -4,11 +4,13 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
@@ -21,22 +23,23 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
 		SiteID:    "test",
 		Provider:  "disqus",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
 
@@ -47,22 +50,23 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	}()
 
 	err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
 		SiteID:    "test",
 		Provider:  "wordpress",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
@@ -76,11 +80,12 @@ func TestMigrator_ImportNative(t *testing.T) {
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
 	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -88,19 +93,20 @@ func TestMigrator_ImportNative(t *testing.T) {
 		SiteID:    "radio-t",
 		Provider:  "native",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	last, err := dataStore.Last("radio-t", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
 
 func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b}
+	defer dataStore.Close()
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -49,7 +49,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	commentsCount := 0
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
-		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
 		if e != nil {
 			return commentsCount, e
 		}
@@ -87,9 +87,49 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
 	return nil
 }
 
+// WithMapper wraps reader with url-mapper.
+func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			log.Printf("[DEBUG] finish write to pipe with %+v", err)
+			if e := w.Close(); e != nil {
+				log.Printf("[WARN] failed close pipe writer with %+v", e)
+			}
+		}()
+
+		// decode from reader and encode to pipe writer
+		dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
+
+		m := meta{}
+		if err = dec.Decode(&m); err != nil {
+			return
+		}
+		for i := range m.Posts {
+			m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
+		}
+		if err = enc.Encode(m); err != nil {
+			return
+		}
+
+		for {
+			comment := store.Comment{}
+			if err = dec.Decode(&comment); err != nil {
+				return
+			}
+			comment.Locator.URL = mapper.URL(comment.Locator.URL)
+			if err = enc.Encode(comment); err != nil {
+				return
+			}
+		}
+	}()
+
+	return r
+}
+
 // Import comments from json strings produced by Remark.Export
 func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
-
 	m := meta{}
 	dec := json.NewDecoder(reader)
 	if err = dec.Decode(&m); err != nil {

backend/app/migrator/native_test.go

@@ -4,15 +4,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -20,11 +20,9 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
-
 func TestNative_Export(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // write 2 comments
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
 	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
@@ -32,33 +30,33 @@ func TestNative_Export(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
-	log.Print(c1)
+	t.Log(c1)
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
-	meta := struct {
+	m := struct {
 		Version int                    `json:"version"`
 		Users   []service.UserMetaData `json:"users"`
 		Posts   []service.PostMetaData `json:"posts"`
 	}{}
 
-	require.NoError(t, dec.Decode(&meta), "decode meta")
+	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(meta.Users))
-	assert.Equal(t, "user1", meta.Users[0].ID)
-	assert.Equal(t, false, meta.Users[0].Blocked.Status)
-	assert.Equal(t, true, meta.Users[0].Verified)
-	assert.Equal(t, "user2", meta.Users[1].ID)
-	assert.Equal(t, true, meta.Users[1].Blocked.Status)
-	assert.Equal(t, false, meta.Users[1].Verified)
+	require.Equal(t, 2, len(m.Users))
+	assert.Equal(t, "user1", m.Users[0].ID)
+	assert.Equal(t, false, m.Users[0].Blocked.Status)
+	assert.Equal(t, true, m.Users[0].Verified)
+	assert.Equal(t, "user2", m.Users[1].ID)
+	assert.Equal(t, true, m.Users[1].Blocked.Status)
+	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(meta.Posts))
-	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
-	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+	require.Equal(t, 1, len(m.Posts))
+	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
+	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
 	comments := [3]store.Comment{}
 
@@ -70,21 +68,22 @@ func TestNative_Export(t *testing.T) {
 }
 
 func TestNative_Import(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
 	assert.Equal(t, "1234", comments[0].ParentID)
 	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
@@ -100,49 +99,93 @@ func TestNative_Import(t *testing.T) {
 	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
 }
 
+func TestNative_ImportWithMapper(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
+	// want to remap comments to https://rdt.c
+	rules := `https://radio-t.com* https://rdt.c*`
+	mapper, err := NewURLMapper(strings.NewReader(rules))
+	assert.NoError(t, err)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	mappedReader := WithMapper(strings.NewReader(inp), mapper)
+
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
+	size, err := r.Import(mappedReader, "radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
 func TestNative_ImportWrongVersion(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
 	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
 	assert.Equal(t, 0, size)
 
 }
 func TestNative_ImportManyWithError(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
 
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
-	for i := 0; i < 1200; i++ {
+	for i := 0; i < 100; i++ {
 		buf.WriteString(fmt.Sprintf(goodRec, i))
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
+	assert.Equal(t, 100, n)
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
+	assert.NoError(t, err)
+	assert.Equal(t, 100, len(comments))
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
+func prep(t *testing.T) (*service.DataStore, func()) {
+
+	testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
@@ -152,7 +195,7 @@ func prep(t *testing.T) *service.DataStore {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
@@ -160,7 +203,10 @@ func prep(t *testing.T) *service.DataStore {
 		User:    store.User{ID: "user2", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	return b
+	return b, func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
 }

backend/app/migrator/wordpress_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -19,17 +20,18 @@ func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
 	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
-	assert.Nil(t, err, "create store")
+	assert.NoError(t, err, "create store")
 
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last(siteID, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
+	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
 
 	c := last[0]
 	assert.Equal(t, "14", c.ID)
@@ -42,14 +44,14 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
 
 	posts, err := dataStore.List(siteID, 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(posts))
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(posts))
 
 	p := posts[0]
 	assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
 
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
 }
 
@@ -61,7 +63,7 @@ func TestWordPress_Convert(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
+	require.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
 
 	exp1 := store.Comment{
 		ID: "13",
@@ -88,7 +90,7 @@ func TestWP_Convert_MD(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported")
+	require.Equal(t, 3, len(comments), "3 comments exported")
 
 	assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
 
@@ -188,7 +190,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/notify/email.go

@@ -0,0 +1,476 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"mime/quotedprintable"
+	"net"
+	"net/smtp"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
+	"github.com/pkg/errors"
+)
+
+// EmailParams contain settings for email notifications
+type EmailParams struct {
+	From                 string // from email address
+	MsgTemplate          string // request message template
+	VerificationSubject  string // verification message subject
+	VerificationTemplate string // verification message template
+	SubscribeURL         string // full subscribe handler URL
+	UnsubscribeURL       string // full unsubscribe handler URL
+
+	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
+}
+
+// SMTPParams contain settings for smtp server connection
+type SMTPParams struct {
+	Host     string        // SMTP host
+	Port     int           // SMTP port
+	TLS      bool          // TLS auth
+	Username string        // user name
+	Password string        // password
+	TimeOut  time.Duration // TCP connection timeout
+}
+
+// Email implements notify.Destination for email
+type Email struct {
+	EmailParams
+	SMTPParams
+
+	smtp       smtpClientCreator
+	msgTmpl    *template.Template // parsed request message template
+	verifyTmpl *template.Template // parsed verification message template
+}
+
+// default email client implementation
+type emailClient struct{ smtpClientCreator }
+
+// smtpClient interface defines subset of net/smtp used by email client
+type smtpClient interface {
+	Mail(string) error
+	Auth(smtp.Auth) error
+	Rcpt(string) error
+	Data() (io.WriteCloser, error)
+	Quit() error
+	Close() error
+}
+
+// smtpClientCreator interface defines function for creating new smtpClients
+type smtpClientCreator interface {
+	Create(SMTPParams) (smtpClient, error)
+}
+
+type emailMessage struct {
+	from    string
+	to      string
+	message string
+}
+
+// msgTmplData store data for message from request template execution
+type msgTmplData struct {
+	UserName          string
+	UserPicture       string
+	CommentText       string
+	CommentLink       string
+	CommentDate       time.Time
+	ParentUserName    string
+	ParentUserPicture string
+	ParentCommentText string
+	ParentCommentLink string
+	ParentCommentDate time.Time
+	PostTitle         string
+	Email             string
+	UnsubscribeLink   string
+	ForAdmin          bool
+}
+
+// verifyTmplData store data for verification message template execution
+type verifyTmplData struct {
+	User         string
+	Token        string
+	Email        string
+	Site         string
+	SubscribeURL string
+}
+
+const (
+	defaultVerificationSubject = "Email verification"
+	defaultEmailTimeout        = 10 * time.Second
+	defaultEmailTemplate       = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+	<style type="text/css">
+		img {
+			max-width: 100%;
+			max-height: 250px;
+			margin: 5px 0;
+			display: block;
+			color: #000;
+		}
+		a {
+			text-decoration: none;
+			color: #0aa;
+		}
+		p {
+			margin: 0 0 12px;
+		}
+		blockquote {
+			margin: 10px 0;
+			padding: 12px 12px 1px 12px;
+			background: rgba(255,255,255,.5)
+		}
+	</style>
+</head>
+<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+<body>
+	<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
+		<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
+        {{- if .ForAdmin}}
+		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+        {{- else }}
+		<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
+        {{- end }}
+		<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
+            {{- if .ParentCommentText}}
+			<div style="margin-bottom: 12px; line-height: 24px; word-break: break-all;">
+				<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; display: inline; vertical-align: middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+				<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
+				<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
+				<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
+			</div>
+			<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
+				{{.ParentCommentText}}
+			</div>
+            {{- end }}
+			<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px;">
+				<div style="margin-bottom: 12px;" line-height: 24px;word-break: break-all;>
+					<img src="{{.UserPicture}}" style="width: 24px; height: 24px; display:inline; vertical-align:middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
+					<span style="font-size: 14px; font-weight: bold; color: #777">{{.UserName}}</span>
+					<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.CommentDate.Format "02.01.2006 at 15:04"}}</span>
+					<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
+				</div>
+				<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">{{.CommentText}}</div>
+			</div>
+		</div>
+		<div style="text-align: center; font-size: 14px; margin-top: 32px;">
+			<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a>{{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}</i>
+			<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;"></div>
+			{{- if .UnsubscribeLink}}
+			<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
+			{{- end }}
+			<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
+			<div style="opacity: 0;font-size: 1;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
+		</div>
+	</div>
+</body>
+</html>
+`
+	defaultEmailVerificationTemplate = `<!DOCTYPE html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width" />
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+	<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
+	<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
+		<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+		<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
+		{{- if .SubscribeURL}}
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
+		<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
+		{{- end }}
+		<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+			<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
+			<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
+			<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+		</div>
+		<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
+	</div>
+</body>
+</html>
+`
+)
+
+// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
+func NewEmail(emailParams EmailParams, smtpParams SMTPParams) (*Email, error) {
+	// set up Email emailParams
+	res := Email{EmailParams: emailParams}
+	if res.MsgTemplate == "" {
+		res.MsgTemplate = defaultEmailTemplate
+	}
+	if res.VerificationTemplate == "" {
+		res.VerificationTemplate = defaultEmailVerificationTemplate
+	}
+	if res.VerificationSubject == "" {
+		res.VerificationSubject = defaultVerificationSubject
+	}
+
+	// set up SMTP emailParams
+	res.smtp = &emailClient{}
+	res.SMTPParams = smtpParams
+	if res.TimeOut <= 0 {
+		res.TimeOut = defaultEmailTimeout
+	}
+
+	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
+		res.Host, res.Username, res.TimeOut)
+
+	// initialize templates
+	var err error
+	if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse message template")
+	}
+	if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
+		return nil, errors.Wrapf(err, "can't parse verification template")
+	}
+	return &res, err
+}
+
+// Send email about comment reply to Request.Email if it's set,
+// also sends email to site administrator if appropriate option is set.
+// Thread safe
+func (e *Email) Send(ctx context.Context, req Request) (err error) {
+	if req.Email == "" {
+		// this means we can't send this request via Email
+		return nil
+	}
+	select {
+	case <-ctx.Done():
+		return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
+	default:
+	}
+	var msg string
+
+	if req.Verification.Token != "" {
+		log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
+		msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+		if err != nil {
+			return err
+		}
+	}
+
+	if req.Comment.ID != "" {
+		if req.parent.User.ID == req.Comment.User.ID && !req.ForAdmin {
+			// don't send anything if if user replied to their own comment
+			return nil
+		}
+		log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
+		msg, err = e.buildMessageFromRequest(req, req.ForAdmin)
+		if err != nil {
+			return err
+		}
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.sendMessage(emailMessage{from: e.From, to: req.Email, message: msg})
+		})
+}
+
+// buildVerificationMessage generates verification email message based on given input
+func (e *Email) buildVerificationMessage(user, email, token, site string) (string, error) {
+	subject := e.VerificationSubject
+	msg := bytes.Buffer{}
+	err := e.verifyTmpl.Execute(&msg, verifyTmplData{
+		User:         user,
+		Token:        token,
+		Email:        email,
+		Site:         site,
+		SubscribeURL: e.SubscribeURL,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build verification message")
+	}
+	return e.buildMessage(subject, msg.String(), email, "text/html", "")
+}
+
+// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
+func (e *Email) buildMessageFromRequest(req Request, forAdmin bool) (string, error) {
+	subject := "New reply to your comment"
+	if forAdmin {
+		subject = "New comment to your site"
+	}
+	if req.Comment.PostTitle != "" {
+		subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
+	}
+
+	token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
+	if err != nil {
+		return "", errors.Wrapf(err, "error creating token for unsubscribe link")
+	}
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if forAdmin {
+		unsubscribeLink = ""
+	}
+
+	commentURLPrefix := req.Comment.Locator.URL + uiNav
+	msg := bytes.Buffer{}
+	tmplData := msgTmplData{
+		UserName:        req.Comment.User.Name,
+		UserPicture:     req.Comment.User.Picture,
+		CommentText:     req.Comment.Text,
+		CommentLink:     commentURLPrefix + req.Comment.ID,
+		CommentDate:     req.Comment.Timestamp,
+		PostTitle:       req.Comment.PostTitle,
+		Email:           req.Email,
+		UnsubscribeLink: unsubscribeLink,
+		ForAdmin:        forAdmin,
+	}
+	// in case of message to admin, parent message might be empty
+	if req.Comment.ParentID != "" {
+		tmplData.ParentUserName = req.parent.User.Name
+		tmplData.ParentUserPicture = req.parent.User.Picture
+		tmplData.ParentCommentText = req.parent.Text
+		tmplData.ParentCommentLink = commentURLPrefix + req.parent.ID
+		tmplData.ParentCommentDate = req.parent.Timestamp
+	}
+	err = e.msgTmpl.Execute(&msg, tmplData)
+	if err != nil {
+		return "", errors.Wrapf(err, "error executing template to build comment reply message")
+	}
+	return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
+}
+
+// buildMessage generates email message to send using net/smtp.Data()
+func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink string) (message string, err error) {
+	addHeader := func(msg, h, v string) string {
+		msg += fmt.Sprintf("%s: %s\n", h, v)
+		return msg
+	}
+	message = addHeader(message, "From", e.From)
+	message = addHeader(message, "To", to)
+	message = addHeader(message, "Subject", subject)
+	message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
+
+	if contentType != "" {
+		message = addHeader(message, "MIME-version", "1.0")
+		message = addHeader(message, "Content-Type", contentType+`; charset="UTF-8"`)
+	}
+
+	if unsubscribeLink != "" {
+		// https://support.google.com/mail/answer/81126 -> "Include option to unsubscribe"
+		message = addHeader(message, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
+		message = addHeader(message, "List-Unsubscribe", "<"+unsubscribeLink+">")
+	}
+
+	message = addHeader(message, "Date", time.Now().Format(time.RFC1123Z))
+
+	buff := &bytes.Buffer{}
+	qp := quotedprintable.NewWriter(buff)
+	if _, err := qp.Write([]byte(body)); err != nil {
+		return "", err
+	}
+	// flush now, must NOT use defer, for small body, defer may cause buff.String() got empty body
+	if err := qp.Close(); err != nil {
+		return "", fmt.Errorf("quotedprintable Write failed: %w", err)
+	}
+	m := buff.String()
+	message += "\n" + m
+	return message, nil
+}
+
+// sendMessage sends messages to server in a new connection, closing the connection after finishing.
+// Thread safe.
+func (e *Email) sendMessage(m emailMessage) error {
+	if e.smtp == nil {
+		return errors.New("sendMessage called without client set")
+	}
+	client, err := e.smtp.Create(e.SMTPParams)
+	if err != nil {
+		return errors.Wrap(err, "failed to make smtp Create")
+	}
+
+	defer func() {
+		if err = client.Quit(); err != nil {
+			log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
+			if err = client.Close(); err != nil {
+				log.Printf("[WARN] can't close smtp connection, %v", err)
+			}
+		}
+	}()
+
+	if err = client.Mail(m.from); err != nil {
+		return errors.Wrapf(err, "bad from address %q", m.from)
+	}
+	if err = client.Rcpt(m.to); err != nil {
+		return errors.Wrapf(err, "bad to address %q", m.to)
+	}
+
+	writer, err := client.Data()
+	if err != nil {
+		return errors.Wrap(err, "can't make email writer")
+	}
+
+	defer func() {
+		if err = writer.Close(); err != nil {
+			log.Printf("[WARN] can't close smtp body writer, %v", err)
+		}
+	}()
+
+	buf := bytes.NewBufferString(m.message)
+	if _, err = buf.WriteTo(writer); err != nil {
+		return errors.Wrapf(err, "failed to send email body to %q", m.to)
+	}
+
+	return nil
+}
+
+// String representation of Email object
+func (e *Email) String() string {
+	return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
+}
+
+// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SMTPParams
+// and returns pointer to it. Thread safe.
+func (s *emailClient) Create(params SMTPParams) (smtpClient, error) {
+	authenticate := func(c *smtp.Client) error {
+		if params.Username == "" || params.Password == "" {
+			return nil
+		}
+		auth := smtp.PlainAuth("", params.Username, params.Password, params.Host)
+		if err := c.Auth(auth); err != nil {
+			return errors.Wrapf(err, "failed to auth to smtp %s:%d", params.Host, params.Port)
+		}
+		return nil
+	}
+
+	var c *smtp.Client
+	srvAddress := fmt.Sprintf("%s:%d", params.Host, params.Port)
+	if params.TLS {
+		tlsConf := &tls.Config{
+			InsecureSkipVerify: false,
+			ServerName:         params.Host,
+		}
+		conn, err := tls.Dial("tcp", srvAddress, tlsConf)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to dial smtp tls to %s", srvAddress)
+		}
+		if c, err = smtp.NewClient(conn, params.Host); err != nil {
+			return nil, errors.Wrapf(err, "failed to make smtp client for %s", srvAddress)
+		}
+		return c, authenticate(c)
+	}
+
+	conn, err := net.DialTimeout("tcp", srvAddress, params.TimeOut)
+	if err != nil {
+		return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
+	}
+
+	c, err = smtp.NewClient(conn, params.Host)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to dial")
+	}
+
+	return c, authenticate(c)
+}

backend/app/notify/email_test.go

@@ -0,0 +1,378 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/smtp"
+	"sync"
+	"testing"
+	"text/template"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestEmailNew(t *testing.T) {
+	var testSet = []struct {
+		name        string
+		err         bool
+		errText     string
+		emailParams EmailParams
+		smtpParams  SMTPParams
+	}{
+		{name: "empty"},
+		{name: "with template parse error",
+			err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				MsgTemplate: "{{",
+			}},
+		{name: "with verification template parse error",
+			err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From:                 "test@from",
+				VerificationTemplate: "{{",
+			},
+			smtpParams: SMTPParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+		{name: "normal creation",
+			err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
+			emailParams: EmailParams{
+				From: "test@from",
+			},
+			smtpParams: SMTPParams{
+				Host:     "test@host",
+				Port:     1000,
+				TLS:      true,
+				Username: "test@username",
+				Password: "test@password",
+				TimeOut:  time.Second,
+			},
+		},
+	}
+	for _, d := range testSet {
+		d := d
+		t.Run(d.name, func(t *testing.T) {
+			email, err := NewEmail(d.emailParams, d.smtpParams)
+
+			if d.err && d.errText == "" {
+				assert.Error(t, err)
+				assert.Nil(t, email)
+			} else if d.err && d.errText != "" {
+				assert.EqualError(t, err, d.errText)
+				assert.Nil(t, email)
+			} else {
+				assert.NoError(t, err)
+				assert.NotNil(t, email, "email returned")
+
+				assert.NotNil(t, email.msgTmpl, "e.template is set")
+				assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
+				assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
+				assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
+				if d.smtpParams.TimeOut == 0 {
+					assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
+				} else {
+					assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
+				}
+				assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
+				assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
+				assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
+				assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
+				assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
+			}
+		})
+	}
+}
+
+func TestEmailSendErrors(t *testing.T) {
+	var err error
+	e := Email{}
+	e.TokenGenFn = TokenGenFn
+
+	e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
+		"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
+	e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
+	assert.NoError(t, err)
+
+	e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
+		"sending message to \"bad@example.org\" aborted due to canceled context")
+
+	e.smtp = &fakeTestSMTP{}
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
+		"error creating token for unsubscribe link: token generation error")
+	e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
+	assert.NoError(t, err)
+}
+
+func TestEmailSend_ExitConditions(t *testing.T) {
+	email, err := NewEmail(EmailParams{}, SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "expecting email returned")
+	// prevent triggering e.autoFlush creation
+	emptyRequest := Request{Comment: store.Comment{ID: "999"}}
+	assert.NoError(t, email.Send(context.Background(), emptyRequest),
+		"Message without parent comment User.Email is not sent and returns nil")
+	requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
+	assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
+		"Message with parent comment User equals comment User is not sent and returns nil")
+}
+
+func TestEmailSendClientError(t *testing.T) {
+	var testSet = []struct {
+		name string
+		smtp *fakeTestSMTP
+		err  string
+	}{
+		{name: "failed to verify receiver", smtp: &fakeTestSMTP{fail: map[string]bool{"mail": true}},
+			err: "bad from address \"\": failed to verify sender"},
+		{name: "failed to verify sender", smtp: &fakeTestSMTP{fail: map[string]bool{"rcpt": true}},
+			err: "bad to address \"\": failed to verify receiver"},
+		{name: "failed to close connection", smtp: &fakeTestSMTP{fail: map[string]bool{"quit": true, "close": true}}},
+		{name: "failed to make email writer", smtp: &fakeTestSMTP{fail: map[string]bool{"data": true}},
+			err: "can't make email writer: failed to send"},
+	}
+	for _, d := range testSet {
+		d := d
+		t.Run(d.name, func(t *testing.T) {
+			e := Email{smtp: d.smtp}
+			if d.err != "" {
+				assert.EqualError(t, e.sendMessage(emailMessage{}), d.err,
+					"expected error for e.sendMessage")
+			} else {
+				assert.NoError(t, e.sendMessage(emailMessage{}),
+					"expected no error for e.sendMessage")
+			}
+		})
+	}
+	e := Email{}
+	e.smtp = nil
+	assert.Error(t, e.sendMessage(emailMessage{}),
+		"nil e.smtp should return error")
+	e.smtp = &fakeTestSMTP{}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage in normal flow")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"quit": true}}
+	assert.NoError(t, e.sendMessage(emailMessage{}), "",
+		"no error expected for e.sendMessage with failed smtpClient.Quit but successful smtpClient.Close")
+	e.smtp = &fakeTestSMTP{fail: map[string]bool{"create": true}}
+	assert.EqualError(t, e.sendMessage(emailMessage{}), "failed to make smtp Create: failed to create client",
+		"e.send called without smtpClient set returns error")
+}
+
+func TestEmail_Send(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSMTP := fakeTestSMTP{}
+	email.smtp = &fakeSMTP
+	email.TokenGenFn = TokenGenFn
+	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
+	req := Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
+		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
+		Email:   "test@example.org",
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildMessageFromRequest(req, req.ForAdmin)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: New reply to your comment for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
+Date: `)
+
+	// send email to admin without parent set
+	req = Request{
+		Comment:  store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Email:    "admin@example.org",
+		ForAdmin: true,
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	res, err = email.buildMessageFromRequest(req, req.ForAdmin)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: admin@example.org
+Subject: New comment to your site for "test_title"
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+}
+
+func TestEmail_SendVerification(t *testing.T) {
+	email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	fakeSMTP := fakeTestSMTP{}
+	email.smtp = &fakeSMTP
+	email.TokenGenFn = TokenGenFn
+	req := Request{
+		Email: "test@example.org",
+		Verification: VerificationMetadata{
+			SiteID: "remark",
+			User:   "test_username",
+			Token:  "secret_",
+		},
+	}
+	assert.NoError(t, email.Send(context.TODO(), req))
+	assert.Equal(t, "from@example.org", fakeSMTP.readMail())
+	assert.Equal(t, 1, fakeSMTP.readQuitCount())
+	assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
+	// test buildMessageFromRequest separately for message text
+	res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `secret_`)
+	assert.NotContains(t, res, `https://example.org/`)
+	email.SubscribeURL = "https://example.org/subscribe.html?token="
+	res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
+	assert.NoError(t, err)
+	assert.Contains(t, res, `From: from@example.org
+To: test@example.org
+Subject: Email verification
+Content-Transfer-Encoding: quoted-printable
+MIME-version: 1.0
+Content-Type: text/html; charset="UTF-8"
+Date: `)
+	assert.Contains(t, res, `https://example.org/subscribe.html?token=3Dsecret_`)
+}
+
+func Test_emailClient_Create(t *testing.T) {
+	creator := emailClient{}
+	client, err := creator.Create(SMTPParams{})
+	assert.Error(t, err, "absence of address to connect results in error")
+	assert.Nil(t, client, "no client returned in case of error")
+}
+
+type fakeTestSMTP struct {
+	fail map[string]bool
+
+	buff       bytes.Buffer
+	mail, rcpt string
+	auth       bool
+	close      bool
+	quitCount  int
+	lock       sync.RWMutex
+}
+
+func (f *fakeTestSMTP) Create(SMTPParams) (smtpClient, error) {
+	if f.fail["create"] {
+		return nil, errors.New("failed to create client")
+	}
+	return f, nil
+}
+
+func (f *fakeTestSMTP) Auth(smtp.Auth) error { f.auth = true; return nil }
+
+func (f *fakeTestSMTP) Mail(m string) error {
+	f.lock.Lock()
+	f.mail = m
+	f.lock.Unlock()
+	if f.fail["mail"] {
+		return errors.New("failed to verify sender")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Rcpt(r string) error {
+	f.lock.Lock()
+	f.rcpt = r
+	f.lock.Unlock()
+	if f.fail["rcpt"] {
+		return errors.New("failed to verify receiver")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Quit() error {
+	f.lock.Lock()
+	f.quitCount++
+	f.lock.Unlock()
+	if f.fail["quit"] {
+		return errors.New("failed to quit")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Close() error {
+	f.close = true
+	if f.fail["close"] {
+		return errors.New("failed to close")
+	}
+	return nil
+}
+
+func (f *fakeTestSMTP) Data() (io.WriteCloser, error) {
+	if f.fail["data"] {
+		return nil, errors.New("failed to send")
+	}
+	return nopCloser{&f.buff}, nil
+}
+
+func (f *fakeTestSMTP) readRcpt() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.rcpt
+}
+
+func (f *fakeTestSMTP) readMail() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.mail
+}
+
+func (f *fakeTestSMTP) readQuitCount() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.quitCount
+}
+
+func TokenGenFn(user, _, _ string) (string, error) {
+	if user == "error" {
+		return "", errors.New("token generation error")
+	}
+	return "token", nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (nopCloser) Close() error {
+	return nil
+}

backend/app/notify/notify.go

@@ -16,7 +16,7 @@ import (
 type Service struct {
 	dataService  Store
 	destinations []Destination
-	queue        chan request
+	queue        chan Request
 
 	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
 	ctx    context.Context
@@ -26,17 +26,30 @@ type Service struct {
 // Destination defines interface for a given destination service, like telegram, email and so on
 type Destination interface {
 	fmt.Stringer
-	Send(ctx context.Context, req request) error
+	Send(ctx context.Context, req Request) error
 }
 
 // Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
-	Get(locator store.Locator, id string) (store.Comment, error)
+	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
 }
 
-type request struct {
-	comment store.Comment
-	parent  store.Comment
+// Request notification either about comment or about particular user verification
+type Request struct {
+	Comment  store.Comment // if set sent notifications about new comment
+	parent   store.Comment // fetched only in case Comment is set
+	Email    string        // if set (also) send email
+	ForAdmin bool          // if set, message supposed to be sent to administrator
+
+	Verification VerificationMetadata // if set sent verification notification
+}
+
+// VerificationMetadata required to send notify method verification message
+type VerificationMetadata struct {
+	SiteID string
+	User   string
+	Token  string
 }
 
 const defaultQueueSize = 100
@@ -50,7 +63,7 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	ctx, cancel := context.WithCancel(context.Background())
 	res := Service{
 		dataService:  dataService,
-		queue:        make(chan request, size),
+		queue:        make(chan Request, size),
 		destinations: destinations,
 		ctx:          ctx,
 		cancel:       cancel,
@@ -62,21 +75,29 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	return &res
 }
 
-// Submit comment to internal channel if not busy, drop if can't send
-func (s *Service) Submit(comment store.Comment) {
+// Submit Request to internal channel if not busy, drop if can't send
+func (s *Service) Submit(req Request) {
 	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
 		return
 	}
-	parentComment := store.Comment{}
-	if s.dataService != nil {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
-			parentComment = p
+	// parent comment is fetched only if comment is present in the Request
+	if s.dataService != nil && req.Comment.ParentID != "" {
+		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
+			req.parent = p
+			// user notification, should fetch email for it.
+			// administrator notification comes with pre-set email
+			if req.Email == "" {
+				req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
+				if err != nil {
+					log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
+				}
+			}
 		}
 	}
 	select {
-	case s.queue <- request{comment: comment, parent: parentComment}:
+	case s.queue <- req:
 	default:
-		log.Printf("[WARN] can't send comment notification to queue, %+v", comment)
+		log.Printf("[WARN] can't send notification to queue, %+v", req.Comment)
 	}
 }
 

backend/app/notify/notify_mock.go

@@ -0,0 +1,43 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// MockDest is a destination mock
+type MockDest struct {
+	data   []Request
+	id     int
+	closed bool
+	lock   sync.Mutex
+}
+
+// Send mock
+func (m *MockDest) Send(ctx context.Context, r Request) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.data = append(m.data, r)
+		log.Printf("sent %s -> %d", r.Comment.ID, m.id)
+	case <-ctx.Done():
+		log.Printf("ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+// Get mock
+func (m *MockDest) Get() []Request {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]Request, len(m.data))
+	copy(res, m.data)
+	return res
+}
+func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }

backend/app/notify/notify_test.go

@@ -1,90 +1,89 @@
 package notify
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"math/rand"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestService_NoDestinations(t *testing.T) {
-	s := NewService(nil, 1)
+	s := NewService(nil, 0)
+	assert.Equal(t, defaultQueueSize, cap(s.queue))
 	assert.NotNil(t, s)
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "101"})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	assert.Equal(t, 3, len(d1.get()), "got all comments to d1")
-	assert.Equal(t, 3, len(d2.get()), "got all comments to d2")
+	require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
+	require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
 
-	assert.Equal(t, "100", d1.get()[0].comment.ID)
-	assert.Equal(t, "101", d1.get()[1].comment.ID)
-	assert.Equal(t, "102", d1.get()[2].comment.ID)
+	assert.Equal(t, "100", d1.Get()[0].Comment.ID)
+	assert.Equal(t, "101", d1.Get()[1].Comment.ID)
+	assert.Equal(t, "102", d1.Get()[2].Comment.ID)
 }
 
 func TestService_WithDrops(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
-	s.Submit(store.Comment{ID: "101"})
-	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
-	time.Sleep(time.Millisecond * 110)
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
+	time.Sleep(time.Millisecond * 11)
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
+	time.Sleep(time.Millisecond * 11)
 	s.Close()
 
-	s.Submit(store.Comment{ID: "111"}) // safe to send after close
+	s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
 
-	assert.Equal(t, 2, len(d1.get()), "one comment dropped from d1")
-	assert.Equal(t, 2, len(d2.get()), "one comment dropped from d2")
+	assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
+	assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
 }
 
 func TestService_Many(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 5, d1, d2)
 	assert.NotNil(t, s)
 
 	for i := 0; i < 10; i++ {
-		s.Submit(store.Comment{ID: fmt.Sprintf("%d", 100+i)})
-		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(200)))
+		s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
+		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
 	}
 	s.Close()
 	time.Sleep(time.Millisecond * 10)
 
-	assert.NotEqual(t, 10, len(d1.get()), "some comments dropped from d1")
-	assert.NotEqual(t, 10, len(d2.get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
 
 	assert.True(t, d1.closed)
 	assert.True(t, d2.closed)
 }
 
 func TestService_WithParent(t *testing.T) {
-	dest := &mockDest{id: 1}
+	dest := &MockDest{id: 1}
 	dataStore := &mockStore{data: map[string]store.Comment{}}
 
 	dataStore.data["p1"] = store.Comment{ID: "p1"}
@@ -93,63 +92,37 @@ func TestService_WithParent(t *testing.T) {
 	s := NewService(dataStore, 1, dest)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "c1", ParentID: "p1"})
+	s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "c11", ParentID: "p11"})
+	s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	destRes := dest.get()
-	assert.Equal(t, 2, len(destRes), "two comment notified")
-	assert.Equal(t, "p1", destRes[0].comment.ParentID)
+	destRes := dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ParentID)
 	assert.Equal(t, "p1", destRes[0].parent.ID)
-	assert.Equal(t, "p11", destRes[1].comment.ParentID)
+	assert.Equal(t, "p11", destRes[1].Comment.ParentID)
 	assert.Equal(t, "", destRes[1].parent.ID)
 }
 
 func TestService_Nop(t *testing.T) {
 	s := NopService
-	s.Submit(store.Comment{})
+	s.Submit(Request{Comment: store.Comment{}})
 	s.Close()
 	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
 }
 
-type mockDest struct {
-	data   []request
-	id     int
-	closed bool
-	lock   sync.Mutex
-}
-
-func (m *mockDest) Send(ctx context.Context, r request) error {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	select {
-	case <-time.After(100 * time.Millisecond):
-		m.data = append(m.data, r)
-		log.Printf("sent %s -> %d", r.comment.ID, m.id)
-	case <-ctx.Done():
-		log.Printf("ctx closed %d", m.id)
-		m.closed = true
-	}
-	return nil
-}
-
-func (m *mockDest) get() []request {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	res := make([]request, len(m.data))
-	copy(res, m.data)
-	return res
-}
-func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
-
 type mockStore struct{ data map[string]store.Comment }
 
-func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
 		return store.Comment{}, errors.New("no such id")
 	}
 	return res, nil
 }
+
+func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
+	return "", errors.New("no such user")
+}

backend/app/notify/telegram.go

@@ -40,7 +40,7 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	if res.timeout == 0 {
 		res.timeout = telegramTimeOut
 	}
-	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
+	log.Printf("[DEBUG] create new telegram notifier for chan %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -85,23 +85,32 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 }
 
 // Send to telegram channel
-func (t *Telegram) Send(ctx context.Context, req request) error {
+func (t *Telegram) Send(ctx context.Context, req Request) error {
+	if req.Comment.ID == "" {
+		// verification request received, send nothing
+		return nil
+	}
+	if req.ForAdmin {
+		// request for administrator received, do nothing with it
+		// as we already sent message on request without this flag set
+		return nil
+	}
 	client := http.Client{Timeout: telegramTimeOut}
-	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
+	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
 
-	from := req.comment.User.Name
-	if req.comment.ParentID != "" {
+	from := req.Comment.User.Name
+	if req.Comment.ParentID != "" {
 		from += " → " + req.parent.User.Name
 	}
 	from = "*" + from + "*"
-	link := fmt.Sprintf("↦ [original comment](%s)", req.comment.Locator.URL+uiNav+req.comment.ID)
-	if req.comment.PostTitle != "" {
-		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
+	link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
+	if req.Comment.PostTitle != "" {
+		link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
 	}
 	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
 		t.apiPrefix, t.token, t.channelID)
 
-	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
+	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.Comment.Orig, link)
 	msg = html.UnescapeString(msg)
 	body := struct {
 		Text string `json:"text"`
@@ -113,10 +122,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/notify/telegram_test.go

@@ -30,14 +30,16 @@ func TestTelegram_New(t *testing.T) {
 	assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
 
 	_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "can't decode response:")
 
 	_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.EqualError(t, err, "unexpected telegram status code 404")
 
 	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
-	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "can't initialize telegram notifications")
+	assert.Contains(t, err.Error(), "dial tcp 127.0.0.1:4321: connect: connection refused")
 
 	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
 	assert.Error(t, err, "empty api url not allowed")
@@ -58,24 +60,25 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NoError(t, err)
 	assert.NotNil(t, tb)
-	c := store.Comment{Text: "some text", ParentID: "1"}
+	c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
 	c.User.Name = "from"
 	cp := store.Comment{Text: "some parent text"}
 	cp.User.Name = "to"
 
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 	c.PostTitle = "test title"
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
 	assert.NoError(t, err)
 
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err, "should failed")
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
-	require.NotNil(t, err)
+	assert.Error(t, err, "should failed")
+	err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
+	require.Error(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())
+	require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
 }
 
 func mockTelegramServer() *httptest.Server {

backend/app/rest/api/admin.go

@@ -9,41 +9,37 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService   *service.DataStore
-	cache         cache.LoadingCache
+	dataService   adminStore
+	cache         LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
+type adminStore interface {
+	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
+	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	IsBlocked(siteID string, userID string) bool
+	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
+	BlockedUsers(siteID string) ([]store.BlockedUser, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
+	SetVerified(siteID string, userID string, status bool) error
+	SetReadOnly(locator store.Locator, status bool) error
+	SetPin(locator store.Locator, commentID string, status bool) error
 }
 
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
@@ -70,7 +66,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+	if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
@@ -86,7 +82,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
 
-	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0, rest.GetUserOrEmpty(r))
 	if err != nil || len(ucomments) == 0 {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
@@ -95,7 +91,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, ucomments[0].User)
 }
 
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
+// GET /deleteme?token=jwt - delete all user comments and details by user's request. Gets info about deleted used from provided token
 // request made GET to allow direct click from the email sent by user
 func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -115,7 +111,13 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
@@ -150,6 +152,13 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
+
+	// delete comments for permanently blocked user.
+	if blockStatus && ttl == time.Duration(0) {
+		if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
+			log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
+		}
+	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
@@ -157,7 +166,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 // GET /blocked?site=siteID - list blocked users
 func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
+	users, err := a.dataService.BlockedUsers(siteID)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
@@ -237,42 +246,3 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

backend/app/rest/api/migrator.go

@@ -11,11 +11,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/migrator"
@@ -24,11 +23,12 @@ import (
 
 // Migrator rest with import and export controllers
 type Migrator struct {
-	Cache             cache.LoadingCache
+	Cache             LoadingCache
 	NativeImporter    migrator.Importer
 	DisqusImporter    migrator.Importer
 	WordPressImporter migrator.Importer
 	NativeExporter    migrator.Exporter
+	URLMapperMaker    migrator.MapperMaker
 	KeyStore          KeyStore
 
 	busy map[string]bool
@@ -40,15 +40,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
@@ -108,7 +99,9 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"status": "import request accepted"})
 }
 
-func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /wait?site=site-id
+// waits for migration operation (import or remap)
+func (m *Migrator) waitCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	timeOut := time.Minute * 15
 	if v := r.URL.Query().Get("timeout"); v != "" {
@@ -162,6 +155,62 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// POST /remap?site=site-id
+// remap urls in comments based on given rules (oldUrl newUrl)
+func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	// create new url-mapper from given rules in body
+	mapper, err := m.URLMapperMaker(r.Body)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
+		return
+	}
+	defer r.Body.Close()
+
+	// start remap procedure with mapper
+	go func() {
+		m.setBusy(siteID, true)
+		defer m.setBusy(siteID, false)
+
+		// do export
+		fh, e := ioutil.TempFile("", "remark42_convert")
+		if e != nil {
+			log.Printf("[WARN] failed to make temp file %+v", e)
+			return
+		}
+		defer func() {
+			if e = os.Remove(fh.Name()); e != nil {
+				log.Printf("[WARN] failed to remove temp file %+v", e)
+			}
+		}()
+		log.Printf("[DEBUG] start export for site=%s", siteID)
+		if _, e = m.NativeExporter.Export(fh, siteID); e != nil {
+			log.Printf("[WARN] export failed with %+v", e)
+			return
+		}
+
+		if _, e = fh.Seek(0, 0); e != nil {
+			log.Printf("[WARN] failed to seek file %+v", e)
+			return
+		}
+
+		log.Printf("[DEBUG] start import for site=%s", siteID)
+		mappedReader := migrator.WithMapper(fh, mapper)
+		size, e := m.NativeImporter.Import(mappedReader, siteID)
+		if e != nil {
+			log.Printf("[WARN] import failed with %+v", e)
+			return
+		}
+
+		m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
+		log.Printf("[DEBUG] convert request completed. site=%s, comments=%d", siteID, size)
+	}()
+
+	render.Status(r, http.StatusAccepted)
+	render.JSON(w, r, R.JSON{"status": "convert request accepted"})
+}
+
 // runImport reads from tmpfile and import for given siteID and provider
 func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	m.setBusy(siteID, true)
@@ -184,7 +233,7 @@ func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
 	}
 	log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, provider)
 
-	fh, err := os.Open(tmpfile)
+	fh, err := os.Open(tmpfile) // nolint
 	if err != nil {
 		log.Printf("[WARN] import failed, %v", err)
 		return

backend/app/rest/api/migrator_test.go

@@ -3,60 +3,65 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/migrator"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
@@ -68,134 +73,157 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
-	assert.Nil(t, err)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=remark42&provider=native", contentType, bodyBuf)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
-	assert.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=wordpress", r)
+	assert.NoError(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native&secret=XYZ", r)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 150; i++ {
+	for i := 0; i < 50; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	client = &http.Client{Timeout: 5 * time.Second}
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
-	recs := []string{}
-	for i := 0; i < 150; i++ {
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev",
+"picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	nRecs := 50
+	recs := make([]string, 0, nRecs)
+	for i := 0; i < nRecs; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
-	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with `nRecs` records
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	client = &http.Client{Timeout: 5 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42&timeout=5ms", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>",
+"user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com",
+"admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah1"},
+"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one",
+"id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,
+"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
+"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
-	require.Nil(t, err)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	waitForImportCompletion(t, ts)
+	waitForMigrationCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
@@ -208,11 +236,11 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 200, resp.StatusCode)
 	require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
 
@@ -222,57 +250,123 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
-	require.Nil(t, err)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42", nil)
+	require.NoError(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+func TestMigrator_Remap(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create 2 comments in https://remark42.com/demo/
+	c1 := store.Comment{Text: "first comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u1"}}
+	_, err := srv.DataService.Create(c1)
+	require.NoError(t, err)
+	c2 := store.Comment{Text: "second comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo/"}, User: store.User{ID: "u2"}}
+	_, err = srv.DataService.Create(c2)
+	require.NoError(t, err)
+
+	// create 1 comment in https://remark42.com/demo-another/
+	c3 := store.Comment{Text: "third comment", Timestamp: time.Now(),
+		Locator: store.Locator{SiteID: "remark42", URL: "https://remark42.com/demo-another/"}, User: store.User{ID: "u3"}}
+	_, err = srv.DataService.Create(c3)
+	require.NoError(t, err)
+
+	// set url https://remark42.com/demo-another/ to be readonly
+	err = srv.DataService.SetMetas("remark42", []service.UserMetaData{}, []service.PostMetaData{{
+		URL:      "https://remark42.com/demo-another/",
+		ReadOnly: true,
+	}})
+	require.NoError(t, err)
+
+	// check that comments created as expected
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// we want remap urls to another domain - www.remark42.com
+	rules := "https://remark42.com/* https://www.remark42.com/*"
+	resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	waitForMigrationCompletion(t, ts)
+
+	// after remap finished we should find comments from new urls
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.True(t, comments.Info.ReadOnly)
+
+	// should find nothing from previous url
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
+	require.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+}
+
+func TestMigrator_RemapReject(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// without admin credentials
+	client := &http.Client{Timeout: 1 * time.Second}
+	rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
+	require.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func waitForMigrationCompletion(t *testing.T, ts *httptest.Server) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/wait?site=remark42", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
 	resp, err := client.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
-}
-
-func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
-	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc := &Migrator{
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
-		Cache:             &cache.Nop{},
-		KeyStore:          adminStore,
-	}
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	teardown = func() {
-		ts.Close()
-		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
-	}
-	return ts, teardown
+	assert.Equal(t, "{\"site_id\":\"remark42\",\"status\":\"completed\"}\n", string(b))
 }
 
 var xmlTestWP = `
@@ -364,7 +458,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/rest/api/rest.go

@@ -5,9 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -19,9 +18,9 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/go-pkgz/rest/logger"
 	"github.com/pkg/errors"
 	"github.com/rakyll/statik/fs"
@@ -40,29 +39,44 @@ type Rest struct {
 
 	DataService      *service.DataStore
 	Authenticator    *auth.Service
-	Cache            cache.LoadingCache
+	Cache            LoadingCache
 	ImageProxy       *proxy.Image
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
 	ImageService     *image.Service
+	Streamer         *Streamer
 
+	AnonVote        bool
 	WebRoot         string
 	RemarkURL       string
+	AdminEmail      string
 	ReadOnlyAge     int
 	SharedSecret    string
 	ScoreThresholds struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter float64
+	UpdateLimiter      float64
+	EmailNotifications bool
+	EmojiEnabled       bool
+	SimpleView         bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	adminService admin
+	pubRest   public
+	privRest  private
+	adminRest admin
+	rssRest   rss
+}
+
+// LoadingCache defines interface for caching
+type LoadingCache interface {
+	Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
+	Flush(req lcw.FlusherRequest)                                        // evict matched records
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
@@ -159,24 +173,17 @@ func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 		Addr:              fmt.Sprintf(":%d", port),
 		Handler:           router,
 		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
-		IdleTimeout:       30 * time.Second,
+		// WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
+		IdleTimeout: 30 * time.Second,
 	}
 }
 
 func (s *Rest) routes() chi.Router {
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
 	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
 
-	s.adminService = admin{
-		dataService:   s.DataService,
-		migrator:      s.Migrator,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
+	s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
 
 	corsMiddleware := cors.New(cors.Options{
 		AllowedOrigins:   []string{"*"},
@@ -189,16 +196,18 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
+		r.Use(middleware.Timeout(5 * time.Second))
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
+		r.Use(middleware.Timeout(5 * time.Second))
 		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
@@ -209,6 +218,7 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
+			rava.Use(middleware.Timeout(5 * time.Second))
 			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
 			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
@@ -216,143 +226,228 @@ func (s *Rest) routes() chi.Router {
 
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(middleware.NoCache)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
-			ropen.Get("/info", s.infoCtrl)
-			ropen.Mount("/img", s.ImageProxy.Routes())
-			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Get("/find", s.pubRest.findCommentsCtrl)
+			ropen.Get("/id/{id}", s.pubRest.commentByIDCtrl)
+			ropen.Get("/comments", s.pubRest.findUserCommentsCtrl)
+			ropen.Get("/last/{limit}", s.pubRest.lastCommentsCtrl)
+			ropen.Get("/count", s.pubRest.countCtrl)
+			ropen.Post("/counts", s.pubRest.countMultiCtrl)
+			ropen.Get("/list", s.pubRest.listCtrl)
+			ropen.Post("/preview", s.pubRest.previewCommentCtrl)
+			ropen.Get("/info", s.pubRest.infoCtrl)
+			ropen.Get("/img", s.ImageProxy.Handler)
+
+			ropen.Route("/rss", func(rrss chi.Router) {
+				rrss.Get("/post", s.rssRest.postCommentsCtrl)
+				rrss.Get("/site", s.rssRest.siteCommentsCtrl)
+				rrss.Get("/reply", s.rssRest.repliesCtrl)
+			})
+
+		})
+
+		// open routes, streams, no send timeout
+		rapi.Route("/stream", func(rstream chi.Router) {
+			rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
+			rstream.Get("/info", s.pubRest.infoStreamCtrl)
+			rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
 		})
 
 		// open routes, cached
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
+			ropen.Get("/picture/{user}/{id}", s.pubRest.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(middleware.Timeout(30 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
-			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
+			rauth.Use(authMiddleware.Auth, matchSiteID, middleware.NoCache, logInfoWithBody)
+			rauth.Get("/user", s.privRest.userInfoCtrl)
+			rauth.Get("/userdata", s.privRest.userAllDataCtrl)
+		})
 
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		// admin routes, require auth and admin users only
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(middleware.Timeout(30 * time.Second))
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly, matchSiteID)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminRest.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminRest.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminRest.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminRest.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminRest.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminRest.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminRest.setPinCtrl)
+			radmin.Get("/blocked", s.adminRest.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminRest.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminRest.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminRest.migrator.exportCtrl)
+			radmin.Post("/import", s.adminRest.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
+			radmin.Post("/remap", s.adminRest.migrator.remapCtrl)
+			radmin.Get("/wait", s.adminRest.migrator.waitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(middleware.Timeout(10 * time.Second))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, matchSiteID)
 			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
-			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
+			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
+			rauth.Post("/comment", s.privRest.createCommentCtrl)
+			rauth.Put("/vote/{id}", s.privRest.voteCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.privRest.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Get("/email", s.privRest.getEmailCtrl)
+			rauth.With(rejectAnonUser).Post("/email/subscribe", s.privRest.sendEmailConfirmationCtrl)
+			rauth.With(rejectAnonUser).Post("/email/confirm", s.privRest.setConfirmedEmailCtrl)
+			rauth.With(rejectAnonUser).Delete("/email", s.privRest.deleteEmailCtrl)
 		})
 
+		// protected routes, anonymous rejected
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(middleware.Timeout(10 * time.Second))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, rejectAnonUser, matchSiteID)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+			rauth.Post("/picture", s.privRest.savePictureCtrl)
 		})
 
 	})
 
-	// respond to /robots.txt with the list of allowed paths
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
-				"/img", "/avatar", "/picture"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
+	// open routes on root level
+	router.Group(func(rroot chi.Router) {
+		rroot.Use(middleware.Timeout(10 * time.Second))
+		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
+		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
+		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
+		rroot.Get("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
+		rroot.Post("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
+	})
 
 	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
+	addFileServer(router, "/web", http.Dir(s.WebRoot), s.Version)
 	return router
 }
 
-func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+func (s *Rest) controllerGroups() (public, private, admin, rss) {
 
-	res = s.adminService.alterComments(comments, r) // apply admin's alteration
-
-	// prepare vote info for client view
-	vote := func(c store.Comment, r *http.Request) store.Comment {
-
-		c.Vote = 0 // default is "none" (not voted)
-
-		user, err := rest.GetUserInfo(r)
-		if err != nil {
-			c.Votes = nil // hide voters list and don't set Vote for non-authed user
-			return c
-		}
-
-		if v, ok := c.Votes[user.ID]; ok {
-			if v {
-				c.Vote = 1
-			} else {
-				c.Vote = -1
-			}
-		}
-
-		c.Votes = nil // hide voters list
-		return c
+	pubGrp := public{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		webRoot:          s.WebRoot,
+		streamer:         s.Streamer,
 	}
 
-	for i, c := range res {
-		c = vote(c, r)
-		res[i] = c
+	privGrp := private{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		authenticator:    s.Authenticator,
+		notifyService:    s.NotifyService,
+		remarkURL:        s.RemarkURL,
+		adminEmail:       s.AdminEmail,
+		anonVote:         s.AnonVote,
 	}
 
-	return res
+	admGrp := admin{
+		dataService:   s.DataService,
+		migrator:      s.Migrator,
+		cache:         s.Cache,
+		authenticator: s.Authenticator,
+		readOnlyAge:   s.ReadOnlyAge,
+	}
+
+	rssGrp := rss{
+		dataService: s.DataService,
+		cache:       s.Cache,
+	}
+
+	return pubGrp, privGrp, admGrp, rssGrp
+}
+
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
+// GET /config?site=siteID - returns configuration
+func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	admins, _ := s.DataService.AdminStore.Admins(siteID)
+	emails, _ := s.DataService.AdminStore.Email(siteID)
+
+	cnf := struct {
+		Version            string   `json:"version"`
+		EditDuration       int      `json:"edit_duration"`
+		MaxCommentSize     int      `json:"max_comment_size"`
+		Admins             []string `json:"admins"`
+		AdminEmail         string   `json:"admin_email"`
+		Auth               []string `json:"auth_providers"`
+		AnonVote           bool     `json:"anon_vote"`
+		LowScore           int      `json:"low_score"`
+		CriticalScore      int      `json:"critical_score"`
+		PositiveScore      bool     `json:"positive_score"`
+		ReadOnlyAge        int      `json:"readonly_age"`
+		MaxImageSize       int      `json:"max_image_size"`
+		EmailNotifications bool     `json:"email_notifications"`
+		EmojiEnabled       bool     `json:"emoji_enabled"`
+		SimpleView         bool     `json:"simple_view"`
+	}{
+		Version:            s.Version,
+		EditDuration:       int(s.DataService.EditDuration.Seconds()),
+		MaxCommentSize:     s.DataService.MaxCommentSize,
+		Admins:             admins,
+		AdminEmail:         emails,
+		LowScore:           s.ScoreThresholds.Low,
+		CriticalScore:      s.ScoreThresholds.Critical,
+		PositiveScore:      s.DataService.PositiveScore,
+		ReadOnlyAge:        s.ReadOnlyAge,
+		MaxImageSize:       s.ImageService.MaxSize,
+		EmailNotifications: s.EmailNotifications,
+		EmojiEnabled:       s.EmojiEnabled,
+		AnonVote:           s.AnonVote,
+		SimpleView:         s.SimpleView,
+	}
+
+	cnf.Auth = []string{}
+	for _, ap := range s.Authenticator.Providers() {
+		cnf.Auth = append(cnf.Auth, ap.Name())
+	}
+
+	if cnf.Admins == nil { // prevent json serialization to nil
+		cnf.Admins = []string{}
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, cnf)
 }
 
 // serves static files from /web or embedded by statik
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
+func addFileServer(r chi.Router, path string, root http.FileSystem, version string) {
 
 	var webFS http.Handler
 
@@ -369,20 +464,22 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	origPath := path
 	webFS = http.StripPrefix(path, webFS)
 	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
+		r.Get(path, http.RedirectHandler(path+"/", http.StatusMovedPermanently).ServeHTTP)
 		path += "/"
 	}
 	path += "*"
 
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// don't show dirs, just serve files
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-				http.NotFound(w, r)
-				return
-			}
-			webFS.ServeHTTP(w, r)
-		}))
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)),
+		middleware.Timeout(10*time.Second),
+		cacheControl(time.Hour, version),
+	).Get(path, func(w http.ResponseWriter, r *http.Request) {
+		// don't show dirs, just serve files
+		if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
+			http.NotFound(w, r)
+			return
+		}
+		webFS.ServeHTTP(w, r)
+	})
 }
 
 func encodeJSONWithHTML(v interface{}) ([]byte, error) {
@@ -448,3 +545,79 @@ func rejectAnonUser(next http.Handler) http.Handler {
 	}
 	return http.HandlerFunc(fn)
 }
+
+// matchSiteID is a middleware rejecting users with mismatch between site param and and User.SiteID
+func matchSiteID(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		// skip for basic auth user
+		if user.Name == "admin" && user.ID == "admin" {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		siteID := r.URL.Query().Get("site")
+		if siteID != "" && user.SiteID != siteID {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
+// cacheControl is a middleware setting cache expiration. Using url+version as etag
+func cacheControl(expiration time.Duration, version string) func(http.Handler) http.Handler {
+
+	etag := func(r *http.Request, version string) string {
+		s := version + ":" + r.URL.String()
+		return store.EncodeID(s)
+	}
+
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+			e := `"` + etag(r, version) + `"`
+			w.Header().Set("Etag", e)
+			w.Header().Set("Cache-Control", "max-age="+strconv.Itoa(int(expiration.Seconds())))
+
+			if match := r.Header.Get("If-None-Match"); match != "" {
+				if strings.Contains(match, e) {
+					w.WriteHeader(http.StatusNotModified)
+					return
+				}
+			}
+			h.ServeHTTP(w, r)
+		}
+		return http.HandlerFunc(fn)
+	}
+}
+
+func parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private.go

@@ -1,30 +1,81 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
+type private struct {
+	dataService      privStore
+	cache            LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	notifyService    *notify.Service
+	authenticator    *auth.Service
+	remarkURL        string
+	adminEmail       string
+	anonVote         bool
+}
+
+type privStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	EditComment(locator store.Locator, commentID string, req service.EditRequest) (comment store.Comment, err error)
+	Vote(req service.VoteReq) (comment store.Comment, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
+	SetUserEmail(siteID string, userID string, value string) (string, error)
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	ValidateComment(c *store.Comment) error
+	IsVerified(siteID string, userID string) bool
+	IsReadOnly(locator store.Locator) bool
+	IsBlocked(siteID string, userID string) bool
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+}
+
+const unsubscribeHTML = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Successfully unsubscribed</p>
+</div>
+</body>
+</html>
+`
+
 // POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
@@ -33,20 +84,26 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	user := rest.MustGetUserInfo(r)
+	if user.ID != "admin" && user.SiteID != comment.Locator.SiteID {
+		rest.SendErrorJSON(w, r, http.StatusForbidden,
+			fmt.Errorf("site mismatch, %q not allowed to post to %s", user.SiteID, comment.Locator.SiteID), "invalid site",
+			rest.ErrCommentValidation)
+		return
+	}
 
 	comment.PrepareUntrusted() // clean all fields user not supposed to set
 	comment.User = user
 	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
 
 	comment.Orig = comment.Text // original comment text, prior to md render
-	if err := s.DataService.ValidateComment(&comment); err != nil {
+	if err := s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 
 	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
+	if s.dataService.IsBlocked(comment.Locator.SiteID, comment.User.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
@@ -56,7 +113,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	id, err := s.DataService.Create(comment)
+	id, err := s.dataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
@@ -66,17 +123,22 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// DataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
+	// dataService modifies comment
+	finalComment, err := s.dataService.Get(comment.Locator, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+	s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
-	if s.NotifyService != nil {
-		s.NotifyService.Submit(finalComment)
+	// user notification
+	if s.notifyService != nil {
+		s.notifyService.Submit(notify.Request{Comment: finalComment})
+	}
+	// admin notification
+	if s.notifyService != nil && s.adminEmail != "" {
+		s.notifyService.Submit(notify.Request{Comment: finalComment, Email: s.adminEmail, ForAdmin: true})
 	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -86,7 +148,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	edit := struct {
 		Text    string
@@ -107,7 +169,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	var currComment store.Comment
 	var err error
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
+	if currComment, err = s.dataService.Get(locator, id, rest.GetUserOrEmpty(r)); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
@@ -119,41 +181,53 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.CommentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
 	}
 
-	res, err := s.DataService.EditComment(locator, id, editReq)
+	res, err := s.dataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
 	if err != nil {
-		code := s.parseError(err, rest.ErrCommentRejected)
+		code := parseError(err, rest.ErrCommentRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
 // GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
+		user.Verified = s.dataService.IsVerified(siteID, user.ID)
+
+		email, err := s.dataService.GetUserEmail(siteID, user.ID)
+		if err != nil {
+			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+		}
+		if len(email) > 0 {
+			user.EmailSubscription = true
+		}
 	}
 
 	render.JSON(w, r, user)
 }
 
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
+	if !s.anonVote && strings.HasPrefix(user.ID, "anonymous_") {
+		http.Error(w, "Access denied", http.StatusForbidden)
+		return
+	}
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
 	log.Printf("[DEBUG] vote for comment %s", id)
@@ -166,23 +240,244 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// check if user blocked
-	if s.adminService.checkBlocked(locator.SiteID, user) {
+	if s.dataService.IsBlocked(locator.SiteID, user.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
+	req := service.VoteReq{
+		Locator:   locator,
+		CommentID: id,
+		UserID:    user.ID,
+		UserIP:    strings.Split(r.RemoteAddr, ":")[0],
+		Val:       vote,
+	}
+	comment, err := s.dataService.Vote(req)
 	if err != nil {
-		code := s.parseError(err, rest.ErrVoteRejected)
+		code := parseError(err, rest.ErrVoteRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
 	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
+// getEmailCtrl gets email address for authenticated user.
+// GET /email?site=siteID
+func (s *private) getEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	address, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// sendEmailConfirmationCtrl gets address and siteID from query, makes confirmation token and sends it to user.
+// GET /email/subscribe?site=siteID&address=someone@example.com
+func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	address := r.URL.Query().Get("address")
+	siteID := r.URL.Query().Get("site")
+	if address == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
+		return
+	}
+	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	}
+	if address == existingAddress {
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
+		return
+	}
+	claims := token.Claims{
+		Handshake: &token.Handshake{ID: user.ID + "::" + address},
+		StandardClaims: jwt.StandardClaims{
+			Audience:  r.URL.Query().Get("site"),
+			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
+			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+			Issuer:    "remark42",
+		},
+	}
+
+	tkn, err := s.authenticator.TokenService().Token(claims)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to make verification token", rest.ErrInternal)
+		return
+	}
+
+	s.notifyService.Submit(
+		notify.Request{
+			Email: address,
+			Verification: notify.VerificationMetadata{
+				SiteID: siteID,
+				User:   user.Name,
+				Token:  tkn,
+			},
+		},
+	)
+
+	render.JSON(w, r, R.JSON{"user": user, "address": address})
+}
+
+// setConfirmedEmailCtrl uses provided token parameter (generated by sendEmailConfirmationCtrl) to set email and add it to user token
+// PUT /email/confirm?site=siteID&tkn=jwt
+func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 || elems[0] != user.ID {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	address := elems[1]
+
+	log.Printf("[DEBUG] set email for user %s", user.ID)
+
+	val, err := s.dataService.SetUserEmail(siteID, user.ID, address)
+	if err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set email for user", code)
+		return
+	}
+
+	// update User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	claims.User.Email = address
+	if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+		return
+	}
+	render.JSON(w, r, R.JSON{"updated": true, "address": val})
+}
+
+// POST/GET /email/unsubscribe.html?site=siteID&tkn=jwt - unsubscribe the user in token from email notifications
+func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
+	tkn := r.URL.Query().Get("tkn")
+	if tkn == "" {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
+		return
+	}
+	siteID := r.URL.Query().Get("site")
+
+	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	if err != nil {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	if s.authenticator.TokenService().IsExpired(confClaims) {
+		rest.SendErrorHTML(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+
+	elems := strings.Split(confClaims.Handshake.ID, "::")
+	if len(elems) != 2 {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
+		return
+	}
+	userID := elems[0]
+	address := elems[1]
+
+	existingAddress, err := s.dataService.GetUserEmail(siteID, userID)
+	if err != nil {
+		log.Printf("[WARN] can't read email for %s, %v", userID, err)
+	}
+	if existingAddress == "" {
+		rest.SendErrorHTML(w, r, http.StatusConflict, errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
+		return
+	}
+	if address != existingAddress {
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
+		return
+	}
+
+	log.Printf("[DEBUG] unsubscribe user %s", userID)
+
+	if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token, if user has the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		log.Printf("[DEBUG] unsubscribed user doesn't have valid JWT token to update %s, %v", userID, err)
+	}
+	if claims.User != nil && claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err := tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHTML))
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, nil)
+	render.HTML(w, r, msg.String())
+}
+
+// DELETE /email?site=siteID - removes user's email
+func (s *private) deleteEmailCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] remove email for user %s", user.ID)
+
+	if err := s.dataService.DeleteUserDetail(siteID, user.ID, engine.UserEmail); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
+		return
+	}
+	// clean User.Email from the token
+	claims, _, err := s.authenticator.TokenService().Get(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
+		return
+	}
+	if claims.User.Email != "" {
+		claims.User.Email = ""
+		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
+			return
+		}
+	}
+	render.JSON(w, r, R.JSON{"deleted": true})
+}
+
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
@@ -213,7 +508,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		comments, errUser := s.dataService.User(siteID, user.ID, 100, i*100, rest.GetUserOrEmpty(r))
 		if errUser != nil {
 			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
@@ -240,7 +535,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 // POST /deleteme?site_id=site - requesting delete of all user info
 // makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	siteID := r.URL.Query().Get("site")
 
@@ -260,18 +555,18 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
-	tokenStr, err := s.Authenticator.TokenService().Token(claims)
+	tokenStr, err := s.authenticator.TokenService().Token(claims)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
-	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
+	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.remarkURL, tokenStr)
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
 // POST /image - save image with form request
-func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 
 	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
@@ -279,14 +574,14 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	file, header, err := r.FormFile("file")
+	file, _, err := r.FormFile("file")
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
-	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	id, err := s.imageService.Save(user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return
@@ -295,37 +590,12 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"id": id})
 }
 
-func (s *Rest) isReadOnly(locator store.Locator) bool {
-	if s.ReadOnlyAge > 0 {
+func (s *private) isReadOnly(locator store.Locator) bool {
+	if s.readOnlyAge > 0 {
 		// check RO by age
-		if info, e := s.DataService.Info(locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+		if info, e := s.dataService.Info(locator, s.readOnlyAge); e == nil && info.ReadOnly {
 			return true
 		}
 	}
-	return s.DataService.IsReadOnly(locator) // ro manually
-}
-
-func (s *Rest) parseError(err error, defaultCode int) (code int) {
-	code = defaultCode
-
-	switch {
-	// voting errors
-	case strings.Contains(err.Error(), "can not vote for his own comment"):
-		code = rest.ErrVoteSelf
-	case strings.Contains(err.Error(), "already voted for"):
-		code = rest.ErrVoteDbl
-	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
-		code = rest.ErrVoteMax
-	case strings.Contains(err.Error(), "minimal score reached for comment"):
-		code = rest.ErrVoteMinScore
-
-	// edit errors
-	case strings.HasPrefix(err.Error(), "too late to edit"):
-		code = rest.ErrCommentEditExpired
-	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
-		code = rest.ErrCommentEditChanged
-
-	}
-
-	return code
+	return s.dataService.IsReadOnly(locator) // ro manually
 }

backend/app/rest/api/rest_public.go

@@ -1,54 +1,99 @@
 package api
 
 import (
+	"bytes"
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
+type public struct {
+	dataService      pubStore
+	cache            LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	streamer         *Streamer
+	webRoot          string
+}
+
+type pubStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	UserCount(siteID, userID string) (int, error)
+	Count(locator store.Locator) (int, error)
+	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+
+	ValidateComment(c *store.Comment) error
+	IsReadOnly(locator store.Locator) bool
+	Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
+}
+
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
 	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
 		sort = "+" + sort[1:]
 	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
+
+	view := r.URL.Query().Get("view")
+	since, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
+		return
+	}
+	format := r.URL.Query().Get("format")
+	if format == "tree" {
+		since = time.Time{} // since doesn't make sense for tree
+	}
+
+	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v", locator, sort, format, since)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, sort)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
 		if e != nil {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.alterComments(comments, r)
+		comments = s.applyView(comments, view)
 		var b []byte
-		switch r.URL.Query().Get("format") {
+		switch format {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(comments, sort, s.readOnlyAge)
 			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*rest.Node{}
+				tree.Nodes = []*service.Node{}
 			}
-			if s.DataService.IsReadOnly(locator) {
+			if s.dataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
 			b, e = encodeJSONWithHTML(tree)
 		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
+			withInfo := commentsWithInfo{Comments: comments}
+			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
 				withInfo.Info = info
 			}
 			b, e = encodeJSONWithHTML(withInfo)
@@ -67,7 +112,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
@@ -81,23 +126,23 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	comment.User = user
 	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
+	if err = s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 	comment.Sanitize()
 	render.HTML(w, r, comment.Text)
 }
 
 // GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		info, e := s.dataService.Info(locator, s.readOnlyAge)
 		if e != nil {
 			return nil, e
 		}
@@ -114,8 +159,52 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
+func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+	log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
+	fn := func() steamEventFn {
+		lastTS := sinceTs
+		lastCount := 0
+
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				info, e := s.dataService.Info(locator, s.readOnlyAge)
+				if e != nil {
+					return nil, e
+				}
+				// cache update used as indication of post update. comparing lastTS for no-cache.
+				// removal won't update lastTS, count check will catch it.
+				if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
+					upd = true
+				}
+				lastTS = info.LastTS
+				lastCount = info.Count
+				return encodeJSONWithHTML(info)
+			})
+			if err != nil {
+				return "info", data, false, err
+			}
+			return "info", data, upd, nil
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
+// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
+// limited with "since" param
+func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get last comments for %s", siteID)
 
@@ -124,13 +213,18 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		limit = 0
 	}
 
+	sinceTime, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Last(siteID, limit, sinceTime, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
@@ -146,8 +240,47 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
+func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] get last comments stream for %s", siteID)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+	if sinceTs.IsZero() {
+		sinceTs = time.Now()
+	}
+
+	fn := func() steamEventFn {
+		sinceTime := sinceTs
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
+				if e != nil {
+					return nil, e
+				}
+				sinceTime = time.Now()
+				if len(comments) > 0 {
+					sinceTime = comments[0].Timestamp
+					upd = true
+				}
+				return encodeJSONWithHTML(comments)
+			})
+			return "last", data, upd, err
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	id := chi.URLParam(r, "id")
 	siteID := r.URL.Query().Get("site")
@@ -155,12 +288,11 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
 
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
+	comment, err := s.dataService.Get(store.Locator{SiteID: siteID, URL: url}, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -169,7 +301,7 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
@@ -187,14 +319,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
 	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.User(siteID, userID, limit, 0, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
+		count, e := s.dataService.UserCount(siteID, userID)
 		if e != nil {
 			return nil, e
 		}
@@ -212,53 +343,10 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		PositiveScore  bool     `json:"positive_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-		MaxImageSize   int      `json:"max_image_size"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.DataService.AdminStore.Admins(siteID),
-		AdminEmail:     s.DataService.AdminStore.Email(siteID),
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		PositiveScore:  s.DataService.PositiveScore,
-		ReadOnlyAge:    s.ReadOnlyAge,
-		MaxImageSize:   s.ImageService.Store.SizeLimit(),
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers() {
-		cnf.Auth = append(cnf.Auth, ap.Name())
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
 // GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
+	count, err := s.dataService.Count(locator)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
@@ -267,7 +355,7 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /counts?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
@@ -277,12 +365,12 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	h := sha1.Sum([]byte(k)) //nolint
+	h := sha1.Sum([]byte(k)) // nolint
 	sha := base64.URLEncoding.EncodeToString(h[:])
 
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		counts, e := s.dataService.Counts(siteID, posts)
 		if e != nil {
 			return nil, e
 		}
@@ -300,7 +388,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 	siteID := r.URL.Query().Get("site")
 	limit, skip := 0, 0
@@ -313,8 +401,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		posts, e := s.dataService.List(siteID, limit, skip)
 		if e != nil {
 			return nil, e
 		}
@@ -327,28 +415,14 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
-		log.Printf("[WARN] can't render posts lits for site %s", siteID)
+		log.Printf("[WARN] can't render posts list for site %s", siteID)
 	}
 }
 
 // GET /picture/{user}/{id} - get picture
-func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
-
-	imgContentType := func(img string) string {
-		img = strings.ToLower(img)
-		switch {
-		case strings.HasSuffix(img, ".png"):
-			return "image/png"
-		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
-			return "image/jpeg"
-		case strings.HasSuffix(img, ".gif"):
-			return "image/gif"
-		}
-		return "image/*"
-	}
-
+func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
-	imgRdr, size, err := s.ImageService.Load(id)
+	img, err := s.imageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
 		return
@@ -364,12 +438,57 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer imgRdr.Close()
-
-	w.Header().Set("Content-Type", imgContentType(id))
-	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.Header().Set("Content-Type", s.imageService.ImgContentType(img))
+	w.Header().Set("Content-Length", strconv.Itoa(len(img)))
 	w.WriteHeader(http.StatusOK)
-	if _, err = io.Copy(w, imgRdr); err != nil {
+	if _, err = io.Copy(w, bytes.NewReader(img)); err != nil {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.webRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/user",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}
+
+func (s *public) applyView(comments []store.Comment, view string) []store.Comment {
+	if strings.EqualFold(view, "user") {
+		projection := make([]store.Comment, len(comments))
+		for i, c := range comments {
+			p := store.Comment{
+				ID:   c.ID,
+				User: c.User,
+			}
+			projection[i] = p
+		}
+		return projection
+	}
+	return comments
+}
+
+func (s *public) parseSince(r *http.Request) (time.Time, error) {
+	sinceTs := time.Time{}
+	if since := r.URL.Query().Get("since"); since != "" {
+		unixTS, e := strconv.ParseInt(since, 10, 64)
+		if e != nil {
+			return time.Time{}, errors.Wrap(e, "can't translate since parameter")
+		}
+		sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+	}
+	return sinceTs, nil
+}

backend/app/rest/api/rest_public_test.go

@@ -1,21 +1,24 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -32,11 +35,15 @@ func TestRest_Preview(t *testing.T) {
 	defer teardown()
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
+
+	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_PreviewWithMD(t *testing.T) {
@@ -56,67 +63,96 @@ BKT
 	text = strings.Replace(text, "BKT", "```", -1)
 	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
 	j = strings.Replace(j, "\n", "\\n", -1)
-	t.Log(j)
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
+	assert.NoError(t, err)
+	assert.Equal(t,
+		`<h1>h1</h1>
+<pre class="chroma">func TestRest_Preview(t *testing.T) {
+srv, ts := prep(t)
+  require.NotNil(t, srv)
+}
+</pre>`,
+		string(b))
+}
+
+func TestRest_PreviewCode(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	text := `BKTgo
+func main(aa string) int {return 0}
+BKT
+`
+	text = strings.Replace(text, "BKT", "```", -1)
+	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+	j = strings.Replace(j, "\n", "\\n", -1)
+
+	resp, err := post(t, ts.URL+"/api/v1/preview", j)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, `<pre class="chroma"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+</pre>`, string(b))
 }
 
 func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id1 := addComment(t, c1, ts)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	id2 := addComment(t, c2, ts)
 
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "<p>test test #1</p>\n", comments.Comments[0].Text)
+	assert.Equal(t, "<p>test test #2</p>\n", comments.Comments[1].Text)
 	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
 	assert.Equal(t, 2, comments.Info.Count)
 	assert.Equal(t, false, comments.Info.ReadOnly)
 	assert.True(t, comments.Info.FirstTS.Before(comments.Info.LastTS))
 
 	// get sorted by -time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=-time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=-time")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[1].ID)
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	tree := service.Tree{}
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(tree.Nodes))
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(tree.Nodes))
 	assert.Equal(t, 1, len(tree.Nodes[0].Replies))
 	assert.Equal(t, 2, tree.Info.Count)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
@@ -128,28 +164,28 @@ func TestRest_FindAge(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is fresh")
 
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is old")
 }
@@ -159,90 +195,148 @@ func TestRest_FindReadOnly(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
 	_, err := srv.DataService.Create(c1)
 
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -2),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	// set post to read-only
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	tree := rest.Tree{}
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
+	tree := service.Tree{}
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
+	tree = service.Tree{}
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
 	assert.False(t, tree.Info.ReadOnly, "post is writable")
 }
 
+func TestRest_FindUserView(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&view=user")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	id1 := addComment(t, c1, ts)
+
+	c2 := store.Comment{Text: "test test #2", ParentID: id1,
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	id2 := addComment(t, c2, ts)
+
+	assert.NotEqual(t, id1, id2)
+
+	// get sorted by +time with view=user
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&sort=+time&view=user")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, id1, comments.Comments[0].ID)
+	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[1].Text)
+}
+
 func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "[]\n", res, "empty last should return empty list")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
+	ts1 := time.Now().UnixNano() / 1000000
 	addComment(t, c1, ts)
 	id1 := addComment(t, c1, ts)
+	time.Sleep(10 * time.Millisecond)
+	ts2 := time.Now().UnixNano() / 1000000
 	id2 := addComment(t, c2, ts)
 
-	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=remark42")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "should have 2 comments")
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts1))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.Equal(t, id1, comments[1].ID)
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=remark42&since=%d", ts.URL, ts2))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(comments), "should have 1 comments")
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	res, code = get(t, ts.URL+"/api/v1/last/X?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/X?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(comments), "should have 3 comments")
 
-	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
-	assert.Nil(t, err)
+	err = srv.DataService.Delete(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
+	assert.NoError(t, err)
 	srv.Cache.Flush(cache.FlusherRequest{})
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/5?site=remark42")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
-	t.Logf("%+v", comments)
+
+	_, code = get(t, ts.URL+"/api/v1/last/2?site=remark42-BLAH")
+	assert.Equal(t, 500, code)
 }
 
 func TestRest_FindUserComments(t *testing.T) {
@@ -250,9 +344,9 @@ func TestRest_FindUserComments(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
 	addComment(t, c1, ts)
@@ -264,10 +358,10 @@ func TestRest_FindUserComments(t *testing.T) {
 	err := srv.DataService.Delete(c2.Locator, id, store.SoftDelete)
 	assert.NoError(t, err)
 
-	_, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=blah")
+	_, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=blah")
 	assert.Equal(t, 400, code, "noting for user blah")
 
-	res, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=dev")
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
 	assert.Equal(t, 200, code)
 
 	resp := struct {
@@ -276,21 +370,26 @@ func TestRest_FindUserComments(t *testing.T) {
 	}{}
 
 	err = json.Unmarshal([]byte(res), &resp)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(resp.Comments), "should have 3 comments")
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(resp.Comments), "should have 3 comments")
 	assert.Equal(t, 4, resp.Count, "should have 3 count")
+
+	// user comment sorted with -time
+	assert.True(t, resp.Comments[0].Timestamp.After(resp.Comments[1].Timestamp))
+	assert.True(t, resp.Comments[1].Timestamp.After(resp.Comments[2].Timestamp))
 }
 
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=radio-t")
+	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=remark42")
 	assert.Equal(t, 200, code)
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, user)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 
 func TestRest_Count(t *testing.T) {
@@ -298,9 +397,9 @@ func TestRest_Count(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -308,18 +407,21 @@ func TestRest_Count(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3.0, j["count"])
 
-	body, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah2")
+	body, code = get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah2")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2.0, j["count"])
+
+	_, code = get(t, ts.URL+"/api/v1/count?site=remark42-BLAH&url=https://radio-t.com/blah1XXX")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_Counts(t *testing.T) {
@@ -327,9 +429,9 @@ func TestRest_Counts(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -337,18 +439,22 @@ func TestRest_Counts(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	j := []store.PostInfo{}
 	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
-		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
+		{URL: "https://radio-t.com/blah2", Count: 2}}, j)
+
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_List(t *testing.T) {
@@ -356,9 +462,9 @@ func TestRest_List(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -366,15 +472,18 @@ func TestRest_List(t *testing.T) {
 	addComment(t, c2, ts)
 	addComment(t, c2, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
 	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
 	assert.Equal(t, 3, pi[1].Count)
+
+	_, code = get(t, ts.URL+"/api/v1/list?site=remark42-BLAH")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_ListWithSkipAndLimit(t *testing.T) {
@@ -382,11 +491,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
 	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah3"}}
 
 	addComment(t, c1, ts)
 	addComment(t, c1, ts)
@@ -396,11 +505,11 @@ func TestRest_ListWithSkipAndLimit(t *testing.T) {
 	addComment(t, c3, ts)
 	addComment(t, c3, ts)
 
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	body, code := get(t, ts.URL+"/api/v1/list?site=remark42&skip=1&limit=2")
 	assert.Equal(t, 200, code)
 	pi := []store.PostInfo{}
 	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	require.Equal(t, 2, len(pi))
 	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
 	assert.Equal(t, 2, pi[0].Count)
@@ -412,13 +521,13 @@ func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
+	body, code := get(t, ts.URL+"/api/v1/config?site=remark42")
 	assert.Equal(t, 200, code)
 	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 300., j["edit_duration"])
-	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
+	assert.EqualValues(t, []interface{}{"a1", "a2"}, j["admins"])
 	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
@@ -426,46 +535,185 @@ func TestRest_Config(t *testing.T) {
 	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
 	assert.Equal(t, 10000., j["max_image_size"])
-	t.Logf("%+v", j)
+	assert.Equal(t, true, j["emoji_enabled"].(bool))
 }
 
 func TestRest_Info(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 
 	user := store.User{ID: "user1", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 
 	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
+	require.NoError(t, err, "%+v", err)
 	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
-	body, code := get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah1")
+	body, code := get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 
 	info := store.PostInfo{}
 	err = json.Unmarshal([]byte(body), &info)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
 		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
 	assert.Equal(t, exp, info)
 
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
+	_, code = get(t, ts.URL+"/api/v1/info?site=remark42-no&url=https://radio-t.com/blah-no")
 	assert.Equal(t, 400, code)
 }
 
+func TestRest_InfoStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 800 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	<-done
+
+	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
+	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
+
+	_, code = get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah123")
+	assert.Equal(t, 500, code)
+}
+
+func TestRest_InfoStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+}
+
+func TestRest_InfoStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_InfoStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 1500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 5; i++ {
+			time.Sleep(300 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1", nil)
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	defer r.Body.Close()
+	<-ctx.Done()
+	<-done
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	recs := strings.Count(string(body), "data:")
+	require.Equal(t, 1, recs, "should have 1 event:\n", string(body))
+	assert.Contains(t, string(body), `"count":2`)
+}
+
+func TestRest_InfoStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 900 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 10; i++ {
+			time.Sleep(15 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1&since=12345678")
+	assert.Equal(t, 200, code)
+	<-done
+	recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
+	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
+}
+
 func TestRest_Robots(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -474,5 +722,194 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/user\nAllow: /api/v1/img\n"+
+		"Allow: /api/v1/avatar\nAllow: /api/v1/picture\n", body)
+}
+
+func TestRest_LastCommentsStream(t *testing.T) {
+	t.Skip() // TODO: enable after cache is migrated to https://github.com/dgraph-io/ristretto
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 50 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			postComment(t, ts.URL)
+			time.Sleep(100 * time.Millisecond)
+		}
+		t.Log("wrote 10 records")
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
+	require.NoError(t, err)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	defer r.Body.Close()
+	<-done
+	body, err := ioutil.ReadAll(r.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 9*3, len(recs), "9 events")
+	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
+}
+
+func TestRest_LastCommentsStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_LastCommentsStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	<-done
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	assert.True(t, len(recs) < 30, "less 10 events")
+}
+
+func TestRest_LastCommentsStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
+	assert.Equal(t, 200, code, "all streams closed, good to go again")
+}
+
+func TestRest_LastCommentsStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	// stream endpoint currently relies on real cache being present
+	cacheBackend, err := cache.NewExpirableCache()
+	require.NoError(t, err)
+	memCache := cache.NewScache(cacheBackend)
+	srv.privRest.cache = memCache
+	srv.pubRest.cache = memCache
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 1; i < 10; i++ {
+			time.Sleep(50 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42&since=123456", nil)
+	require.NoError(t, err)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	<-done
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "should be 10 events, including first record:\n", recs)
+}
+
+func postComment(t *testing.T, url string) {
+	resp, err := post(t, url+"/api/v1/comment",
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
 }

backend/app/rest/api/rest_test.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -15,16 +17,17 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
+	cache "github.com/go-pkgz/lcw"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
@@ -34,28 +37,36 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testHTML = "/tmp/test-remark.html"
-var getStartedHTML = "/tmp/getstarted.html"
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
 
-var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
 
-var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
+
+var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	body, code := get(t, ts.URL+"/web/test-remark.html")
+	testHTMLName := "test-remark.html"
+	testHTMLFile := os.TempDir() + "/" + testHTMLName
+	err := ioutil.WriteFile(testHTMLFile, []byte("some html"), 0700)
+	assert.NoError(t, err)
+
+	body, code := get(t, ts.URL+"/web/"+testHTMLName)
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "some html", body)
+	_ = os.Remove(testHTMLFile)
 }
 
 func TestRest_GetStarted(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
+	getStartedHTML := os.TempDir() + "/getstarted.html"
 	err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	body, code := get(t, ts.URL+"/index.html")
 	assert.Equal(t, 200, code)
@@ -69,15 +80,20 @@ func TestRest_GetStarted(t *testing.T) {
 
 func TestRest_Shutdown(t *testing.T) {
 	srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
+	done := make(chan bool)
 
+	// without waiting for channel close at the end goroutine will stay alive after test finish
+	// which would create data race with next test
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(200 * time.Millisecond)
 		srv.Shutdown()
+		close(done)
 	}()
 
 	st := time.Now()
 	srv.Run(0)
 	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
+	<-done
 }
 
 func TestRest_filterComments(t *testing.T) {
@@ -96,6 +112,7 @@ func TestRest_filterComments(t *testing.T) {
 }
 
 func TestRest_RunStaticSSLMode(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: auth.NewService(auth.Opts{
 			AvatarStore:       avatar.NewLocalFS("/tmp"),
@@ -105,18 +122,19 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		ImageProxy: &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Static,
-			Port:    8443,
+			Port:    sslPort,
 			Key:     "../../cmd/testdata/key.pem",
 			Cert:    "../../cmd/testdata/cert.pem",
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
-		srv.Run(38080)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -130,40 +148,42 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38080/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
-	resp, err = client.Get("https://localhost:8443/ping")
-	require.Nil(t, err)
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	srv.Shutdown()
 }
 
 func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
+	sslPort := chooseRandomUnusedPort()
 	srv := Rest{
 		Authenticator: &auth.Service{},
 		ImageProxy:    &proxy.Image{},
 		SSLConfig: SSLConfig{
 			SSLMode: Auto,
-			Port:    8443,
+			Port:    sslPort,
 		},
-		RemarkURL: "https://localhost:8443",
+		RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
 	}
 
+	port := chooseRandomUnusedPort()
 	go func() {
 		// can't check https server locally, just only http server
-		srv.Run(38081)
+		srv.Run(port)
 	}()
 
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -172,11 +192,11 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 		},
 	}
 
-	resp, err := client.Get("http://localhost:38081/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	srv.Shutdown()
 }
@@ -213,6 +233,7 @@ func Test_URLKey(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -238,6 +259,7 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 	for i, tt := range tbl {
+		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			r, err := http.NewRequest("GET", tt.url, nil)
 			require.NoError(t, err)
@@ -249,28 +271,91 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 }
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	for n, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}
+
+func TestRest_cacheControl(t *testing.T) {
+
+	tbl := []struct {
+		url     string
+		version string
+		exp     time.Duration
+		etag    string
+		maxAge  int
+	}{
+		{"http://example.com/foo", "v1", time.Hour, "b433be1ea19edaee9dc92ca4b895b6bdf3c058cb", 3600},
+		{"http://example.com/foo2", "v1", 10 * time.Hour, "6d8466aef3246c1057452561acddf7ad9d0d99e0", 36000},
+		{"http://example.com/foo", "v2", time.Hour, "481700c52aab0dfbca99f3ffc2a4fbb27884c114", 3600},
+		{"https://example.com/foo", "v2", time.Hour, "bebd4f1b87f474792c4e75e5affe31fbf67f5778", 3600},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := httptest.NewRequest("GET", tt.url, nil)
+			w := httptest.NewRecorder()
+
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h.ServeHTTP(w, req)
+			resp := w.Result()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			t.Logf("%+v", resp.Header)
+			assert.Equal(t, `"`+tt.etag+`"`, resp.Header.Get("Etag"))
+			assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds())), resp.Header.Get("Cache-Control"))
+
+		})
+	}
+
+}
+
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	tmp := os.TempDir()
+	var testDb string
+	// pick a file name which is not in use for sure
+	for i := 0; i < 10; i++ {
+		testDb = fmt.Sprintf("/%s/test-remark-%d.db", tmp, rand.Int31())
+		_, err := os.Stat(testDb)
+		if err != nil {
+			break
+		}
+	}
+	_ = os.RemoveAll(tmp + "/ava-remark42")
+	_ = os.RemoveAll(tmp + "/pics-remark42")
 
-	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
-	os.Remove(testDb)
-	os.Remove(testHTML)
-	os.RemoveAll("/tmp/ava-remark42")
-	os.RemoveAll("/tmp/pics-remark42")
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
+	require.NoError(t, err)
 
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
+	memCache := cache.NewScache(cache.NewNopCache())
 
-	memCache, err := cache.NewMemoryCache()
-	assert.NoError(t, err)
-
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
-		Interface:              b,
+		Engine:                 b,
 		EditDuration:           5 * time.Minute,
 		MaxCommentSize:         4000,
-		AdminStore:             adminStore,
+		AdminStore:             astore,
 		MaxVotes:               service.UnlimitedVotes,
 		RestrictedWordsMatcher: restrictedWordsMatcher,
 	}
@@ -279,21 +364,21 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
-			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
+			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
+			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
+		}),
+		Cache:      memCache,
+		WebRoot:    tmp,
+		RemarkURL:  "https://demo.remark42.com",
+		AdminEmail: "admin@example.org",
+		ImageService: image.NewService(&image.FileSystem{
+			Location:   tmp + "/pics-remark42",
+			Partitions: 100,
+			Staging:    tmp + "/pics-remark42/staging",
+		}, image.ServiceParams{
+			EditDuration: 100 * time.Millisecond,
+			MaxSize:      10000,
 		}),
-		Cache:     memCache,
-		WebRoot:   "/tmp",
-		RemarkURL: "https://demo.remark42.com",
-		ImageService: &image.Service{
-			Store: &image.FileSystem{
-				Location:   "/tmp/pics-remark42",
-				Partitions: 100,
-				MaxSize:    10000,
-				Staging:    "/tmp/pics-remark42/staging",
-			},
-			TTL: time.Millisecond * 100,
-		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -302,30 +387,34 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
-			Cache:             &cache.Nop{},
-			KeyStore:          adminStore,
+			URLMapperMaker:    migrator.NewURLMapper,
+			Cache:             memCache,
+			KeyStore:          astore,
 		},
+		Streamer: &Streamer{
+			Refresh:   100 * time.Millisecond,
+			TimeOut:   5 * time.Second,
+			MaxActive: 100,
+		},
+		NotifyService: notify.NopService,
+		EmojiEnabled:  true,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-
 	ts = httptest.NewServer(srv.routes())
 
 	teardown = func() {
 		ts.Close()
 		require.NoError(t, srv.DataService.Close())
-		os.Remove(testDb)
-		os.Remove(testHTML)
-		os.RemoveAll("/tmp/ava-remark42")
-		os.RemoveAll("/tmp/pics-remark42")
+		_ = os.Remove(testDb)
+		_ = os.RemoveAll(tmp + "/ava-remark42")
+		_ = os.RemoveAll(tmp + "/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
-// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+// fake auth middleware make user authenticated and uses query's fake_id for ID and fake_name for Name
 func fakeAuth(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Query().Get("fake_id") != "" {
@@ -341,14 +430,14 @@ func fakeAuth(next http.Handler) http.Handler {
 
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	return string(body), r.StatusCode
 }
 
-func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
 	client := http.Client{Timeout: 5 * time.Second}
 	if token != "" {
 		r.Header.Set("X-JWT", token)
@@ -359,53 +448,86 @@ func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	b, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(b), r.StatusCode
 }
 
 func getWithAdminAuth(t *testing.T, url string) (string, int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	r, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer r.Body.Close()
 	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	return string(body), r.StatusCode
 }
 func post(t *testing.T, url string, body string) (*http.Response, error) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	return client.Do(req)
 }
 
 func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	b, err := json.Marshal(c)
-	require.Nil(t, err, "can't marshal comment %+v", c)
+	require.NoError(t, err, "can't marshal comment %+v", c)
 
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	crResp := R.JSON{}
 	err = json.Unmarshal(b, &crResp)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}

backend/app/rest/api/rss.go

@@ -5,9 +5,8 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
 	"github.com/pkg/errors"
 
@@ -15,38 +14,40 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
+type rss struct {
+	dataService rssStore
+	cache       LoadingCache
+}
+
+type rssStore interface {
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error)
+}
+
 const maxRssItems = 20
-const maxLastCommentsReply = 5000
 const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[DEBUG] get rss for post %+v", locator)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Find(locator, "-time", rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
+		feed, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -63,23 +64,22 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) siteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss for site %s", siteID)
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.Last(siteID, maxRssItems, time.Time{}, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
+		feed, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -95,43 +95,24 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/reply?user=userID&site=siteID
-func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) repliesCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
-	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() (res []byte, e error) {
-		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+	data, err := s.cache.Get(key, func() (res []byte, e error) {
+
+		replies, userName, e := s.dataService.UserReplies(siteID, userID, maxRssItems, maxReplyDuration)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.alterComments(comments, r)
-		replies := []store.Comment{}
-		for _, c := range comments {
-			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
-				break
-			}
-			if c.User.ID != userID {
-				userName = c.User.Name
-			}
-			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
-				var pc store.Comment
-				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
-					return nil, errors.Wrap(e, "can't get parent comment")
-				}
-				if pc.User.ID == userID {
-					replies = append(replies, c)
-				}
-			}
-		}
 
-		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
+		feed, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -146,7 +127,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
+func (s *rss) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
 	if description == "" {
 		description = "comment updates"
@@ -175,9 +156,10 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment, description strin
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
+			parentComment, err := s.dataService.Get(c.Locator, c.ParentID, store.User{})
 			if err == nil {
 				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
+				f.Description = f.Description + "<blockquote><p>" + parentComment.Snippet(300) + "</p></blockquote>"
 			} else {
 				log.Printf("[WARN] failed to get info about parent comment, %s", err)
 			}

backend/app/rest/api/rss_test.go

@@ -22,7 +22,7 @@ func TestServer_RssPost(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "1234567890",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	id1, err := rst.DataService.Create(c1)
@@ -30,7 +30,7 @@ func TestServer_RssPost(t *testing.T) {
 	assert.Equal(t, "1234567890", id1)
 	pubDate := time.Now().Format(time.RFC1123Z)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -54,7 +54,7 @@ func TestServer_RssPost(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
+	_, code = get(t, ts.URL+"/api/v1/rss/post?site=remark42-bad&url=https://radio-t.com/blah1")
 	assert.Equal(t, 400, code)
 }
 
@@ -69,13 +69,13 @@ func TestServer_RssSite(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:      "comment-id-2",
 		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -85,15 +85,15 @@ func TestServer_RssSite(t *testing.T) {
 	require.NoError(t, err)
 
 	require.NoError(t, err)
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/site?site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 		  <channel>
 		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
-		    <description>site comment for radio-t</description>
+		    <link>remark42</link>
+		    <description>site comment for remark42</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -132,14 +132,14 @@ func TestServer_RssWithReply(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-id-1",
 		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-id-2",
 		ParentID: "comment-id-1",
 		Text:     "xyz test",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
 		User:     store.User{ID: "u1", Name: "developer one"},
 	}
 
@@ -148,7 +148,7 @@ func TestServer_RssWithReply(t *testing.T) {
 	_, err = rst.DataService.Create(c2)
 	require.NoError(t, err)
 
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah10")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 
@@ -161,7 +161,7 @@ func TestServer_RssWithReply(t *testing.T) {
 		    <item>
 		      <title>developer one &gt; developer one</title>
 		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
-		      <description>xyz test</description>
+              <description>xyz test&lt;blockquote&gt;&lt;p&gt;test 123&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>developer one</author>
               <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
@@ -192,34 +192,34 @@ func TestServer_RssReplies(t *testing.T) {
 	c1 := store.Comment{
 		ID:      "comment-1",
 		Text:    "c1",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "user1", Name: "user1"},
 	}
 	c2 := store.Comment{
 		ID:       "comment-2",
 		Text:     "reply to c1 from user2",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user2", Name: "user2"},
 	}
 	c3 := store.Comment{
 		ID:       "comment-3",
 		Text:     "reply to c1 from user3",
 		ParentID: "comment-1",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "user3", Name: "user3"},
 	}
 	c4 := store.Comment{
 		ID:       "comment-4",
 		Text:     "reply to c2 from developer one",
 		ParentID: "comment-2",
-		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:     store.User{ID: "dev", Name: "developer one"},
 	}
 	c5 := store.Comment{
 		ID:      "comment-5",
 		Text:    "developer one",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
 		User:    store.User{ID: "dev", Name: "developer one"},
 	}
 
@@ -235,19 +235,19 @@ func TestServer_RssReplies(t *testing.T) {
 	require.NoError(t, err)
 
 	// replies to c1 (user1). Must be [c3, c2]
-	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t")
+	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42")
 	assert.Equal(t, 200, code)
 	t.Log(res)
 	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
 	    <channel>
 	        <title>Remark42 comments</title>
-	        <link>radio-t</link>
-	        <description>replies to user2</description>
+	        <link>remark42</link>
+	        <description>replies to user1</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
-		      <description>reply to c1 from user3</description>
+		      <description>reply to c1 from user3&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user3</author>
               <guid>comment-3</guid> 
 		      <pubDate>%s</pubDate>
@@ -255,7 +255,7 @@ func TestServer_RssReplies(t *testing.T) {
 			<item>
 		      <title>user2 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
-		      <description>reply to c1 from user2</description>
+		      <description>reply to c1 from user2&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user2</author>
               <guid>comment-2</guid> 
 		      <pubDate>%s</pubDate>
@@ -265,7 +265,7 @@ func TestServer_RssReplies(t *testing.T) {
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 
-	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t-bad")
+	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=remark42-bad")
 	assert.Equal(t, 400, code)
 }
 

backend/app/rest/api/ssl.go

@@ -86,7 +86,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager {
 // makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
 func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	cfg := makeTLSConfig()
+	cfg := s.makeTLSConfig()
 	cfg.GetCertificate = m.GetCertificate
 	server.TLSConfig = cfg
 	return server
@@ -95,7 +95,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocer
 // makeHTTPSServer makes https server for static mode
 func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	server.TLSConfig = makeTLSConfig()
+	server.TLSConfig = s.makeTLSConfig()
 	return server
 }
 
@@ -109,7 +109,7 @@ func (s *Rest) getRemarkHost() string {
 	return u.Hostname()
 }
 
-func makeTLSConfig() *tls.Config {
+func (s *Rest) makeTLSConfig() *tls.Config {
 	return &tls.Config{
 		PreferServerCipherSuites: true,
 		CipherSuites: []uint16{
@@ -119,7 +119,7 @@ func makeTLSConfig() *tls.Config {
 			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			// tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
 		},
 		MinVersion: tls.VersionTLS12,
 		CurvePreferences: []tls.CurveID{

backend/app/rest/api/ssl_test.go

@@ -33,7 +33,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
@@ -62,28 +62,28 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	// check http to https redirect response
 	resp, err := client.Get(ts.URL + "/blah?param=1")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 307, resp.StatusCode)
 	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
 
 	// check acme http challenge
 	req, err := http.NewRequest("GET", ts.URL+"/.well-known/acme-challenge/token123", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	req.Host = "localhost" // for passing hostPolicy check
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 404, resp.StatusCode)
 
 	err = m.Cache.Put(context.Background(), "token123+http-01", []byte("token"))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "token", string(body))
 }

backend/app/rest/api/stream.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// Streamer creates endless stream of \n separated json records send to remote client
+type Streamer struct {
+	TimeOut     time.Duration
+	Refresh     time.Duration
+	MaxActive   int32
+	activeCount int32
+}
+
+type steamEventFn func() (event string, data []byte, upd bool, err error)
+
+type steamEventResp struct {
+	data  []byte
+	event string
+	err   error
+}
+
+// Activate starts blocking function streaming update created by eventFn to ResponseWriter
+// canceled on context or inactivity timeout
+// note: eventFn is a closure needed to allow state management inside eventFn
+func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
+	updCh := s.eventsCh(ctx, eventFn())
+
+	count := atomic.AddInt32(&s.activeCount, 1)
+	defer atomic.AddInt32(&s.activeCount, -1)
+	if count > s.MaxActive {
+		return errors.New("too many streams")
+	}
+
+	if ww, ok := w.(http.ResponseWriter); ok {
+		ww.Header().Set("Content-Type", "text/event-stream")
+		ww.Header().Set("Connection", "keep-alive")
+		ww.Header().Set("Cache-Control", "no-cache")
+	}
+
+	for {
+		select {
+		case <-ctx.Done(): // request closed by remote client
+			log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
+			return nil
+		case <-time.After(s.TimeOut): // request closed by timeout
+			log.Printf("[DEBUG] stream closed due to timeout")
+			return nil
+		case resp, ok := <-updCh: // new update
+			if !ok { // closed updCh
+				return nil
+			}
+			if resp.err != nil {
+				return resp.err
+			}
+
+			// make server-sent event record
+			// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+			if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
+				return errors.Wrap(e, "send to stream failed")
+			}
+			if fw, okFlush := w.(http.Flusher); okFlush {
+				fw.Flush()
+			}
+		}
+	}
+}
+
+// populate updates to chan, break on context close
+func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
+	ch := make(chan steamEventResp)
+	go func() {
+		tick := time.NewTicker(s.Refresh)
+		defer func() {
+			close(ch)
+			tick.Stop()
+		}()
+		for {
+			select {
+			case <-ctx.Done(): // request closed by remote client
+				return
+			case <-tick.C:
+				event, resp, upd, err := fn()
+				if err != nil {
+					ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
+					return
+				}
+				if upd {
+					ch <- steamEventResp{event: event, data: resp, err: nil}
+				}
+			}
+		}
+	}()
+	return ch
+}

backend/app/rest/api/stream_test.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStream_Timeout(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 || n > 10 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	err := s.Activate(context.Background(), eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}
+
+func TestStream_Cancel(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err := s.Activate(ctx, eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}

backend/app/rest/httperrors.go

@@ -1,7 +1,10 @@
 package rest
 
 import (
+	"bytes"
 	"fmt"
+	"html/template"
+	"io"
 	"net/http"
 	"net/url"
 	"runtime"
@@ -35,9 +38,51 @@ const (
 	ErrAssetNotFound      = 18 // requested file not found
 )
 
+const errorHTML = `<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+    <h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">{{.Error}}: {{.Details}}.</p>
+</div>
+</body>
+</html>
+`
+
+// errTmplData store data for error message
+type errTmplData struct {
+	Error   string
+	Details string
+}
+
+// SendErrorHTML makes html body with provided template and responds with provided http status code,
+// error code is not included in render as it is intended for UI developers and not for the users
+func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+	// MustExecute behaves like template.Execute, but panics if an error occurs.
+	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
+		if err = tmpl.Execute(wr, data); err != nil {
+			panic(err)
+		}
+	}
+
+	tmpl := template.Must(template.New("error").Parse(errorHTML))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	render.Status(r, httpStatusCode)
+	msg := bytes.Buffer{}
+	MustExecute(tmpl, &msg, errTmplData{
+		Error:   err.Error(),
+		Details: details,
+	})
+	render.HTML(w, r, msg.String())
+}
+
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
 func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
 	render.Status(r, httpStatusCode)
 	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
@@ -56,14 +101,10 @@ func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details strin
 	if pc, file, line, ok := runtime.Caller(2); ok {
 		fnameElems := strings.Split(file, "/")
 		funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-		srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
+		srcFileInfo = fmt.Sprintf("[%s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
 			line, funcNameElems[len(funcNameElems)-1])
 	}
 
-	remoteIP := r.RemoteAddr
-	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
-		remoteIP = remoteIP[:pos]
-	}
-	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
-		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s",
+		details, err, httpStatusCode, errCode, uinfoStr, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -26,23 +27,51 @@ func TestSendErrorJSON(t *testing.T) {
 	defer ts.Close()
 
 	resp, err := http.Get(ts.URL + "/error")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
 	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
+func TestSendErrorHTML(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/error" {
+			t.Log("http err request", r.URL)
+			SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL + "/error")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 500, resp.StatusCode)
+
+	assert.NotContains(t, string(body), `987`, "user html should not contain internal error code")
+	assert.Contains(t, string(body), `error details 123456`)
+	assert.Contains(t, string(body), `error 500`)
+}
+
 func TestErrorDetailsMsg(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		req.RemoteAddr = "1.2.3.4"
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (123) - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsg]")
 	}
 	callerFn()
 }
@@ -50,11 +79,14 @@ func TestErrorDetailsMsg(t *testing.T) {
 func TestErrorDetailsMsgWithUser(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
+		require.NoError(t, err)
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
-		require.Nil(t, err)
+		require.NoError(t, err)
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		assert.Contains(t, msg, "error details 123456 - error 500 - 500 (34567) - test/id - https://example.com/test?k1=v1&k2=v2 - [app/rest/httperrors_test.go:")
+		// error line in the middle of the message is not checked
+		assert.Contains(t, msg, " rest.TestErrorDetailsMsgWithUser]")
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,119 +1,58 @@
 package proxy
 
 import (
+	"bytes"
 	"context"
 	"encoding/base64"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // Image extracts image src from comment's html and provides proxy for them
 // this is needed to keep remark42 running behind of HTTPS serve all images via https
 type Image struct {
-	RemarkURL string
-	RoutePath string
-	Enabled   bool
-	Timeout   time.Duration
+	RemarkURL     string
+	RoutePath     string
+	HTTP2HTTPS    bool
+	CacheExternal bool
+	Timeout       time.Duration
+	ImageService  *image.Service
 }
 
-// Convert all img src links without https to proxied links
+// Convert img src links to proxied links depends on enabled options
 func (p Image) Convert(commentHTML string) string {
-	if !p.Enabled || strings.HasPrefix(p.RemarkURL, "http://") {
-		return commentHTML
+	if p.CacheExternal {
+		imgs, err := p.extract(commentHTML, func(img string) bool { return !strings.HasPrefix(img, p.RemarkURL) })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
 	}
 
-	imgs, err := p.extract(commentHTML)
-	if err != nil {
-		return commentHTML
+	if p.HTTP2HTTPS && !strings.HasPrefix(p.RemarkURL, "http://") {
+		imgs, err := p.extract(commentHTML, func(img string) bool { return strings.HasPrefix(img, "http://") })
+		if err != nil {
+			return commentHTML
+		}
+		commentHTML = p.replace(commentHTML, imgs)
 	}
 
-	return p.replace(commentHTML, imgs)
+	return commentHTML
 }
 
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
-	if !p.Enabled {
-		return router
-	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-			return
-		}
-
-		timeout := 60 * time.Second // default
-		if p.Timeout > 0 {
-			timeout = p.Timeout
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		defer cancel()
-
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-			var e error
-			req, e := http.NewRequest("GET", string(src), nil)
-			if e != nil {
-				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-			}
-			resp, e = client.Do(req.WithContext(ctx))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
-
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
-
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
-		}
-	})
-	return router
-}
-
-// extract gets all non-https images and return list of src
-func (p Image) extract(commentHTML string) ([]string, error) {
+// extract gets all images matching predicate and return list of src
+func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]string, error) {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return nil, errors.Wrap(err, "can't create document")
@@ -121,7 +60,7 @@ func (p Image) extract(commentHTML string) ([]string, error) {
 	result := []string{}
 	doc.Find("img").Each(func(i int, s *goquery.Selection) {
 		if im, ok := s.Attr("src"); ok {
-			if strings.HasPrefix(im, "http://") {
+			if imgSrcPred(im) {
 				result = append(result, im)
 			}
 		}
@@ -131,7 +70,6 @@ func (p Image) extract(commentHTML string) ([]string, error) {
 
 // replace img links in commentHTML with route to proxy, base64 encoded original link
 func (p Image) replace(commentHTML string, imgs []string) string {
-
 	for _, img := range imgs {
 		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
 		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
@@ -140,3 +78,95 @@ func (p Image) replace(commentHTML string, imgs []string) string {
 
 	return commentHTML
 }
+
+// Handler returns http handler respond to proxied request
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
+		return
+	}
+
+	imgURL := string(src)
+	var img []byte
+	imgID, err := image.CachedImgID(imgURL)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse image url "+imgURL, rest.ErrAssetNotFound)
+		return
+	}
+	// try to load from cache for case it was saved when CacheExternal was enabled
+	img, _ = p.ImageService.Load(imgID)
+	if img == nil {
+		img, err = p.downloadImage(context.Background(), imgURL)
+		if err != nil {
+			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
+			return
+		}
+		if p.CacheExternal {
+			p.cacheImage(bytes.NewReader(img), imgID)
+		}
+	}
+
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	w.Header().Add("Content-Type", p.ImageService.ImgContentType(img))
+	_, err = io.Copy(w, bytes.NewReader(img))
+	if err != nil {
+		log.Printf("[WARN] can't copy image stream, %s", err)
+	}
+}
+
+// cache image from provided Reader using given ID
+func (p Image) cacheImage(r io.Reader, imgID string) {
+	err := p.ImageService.SaveWithID(imgID, r)
+	if err != nil {
+		log.Printf("[WARN] unable to save image to the storage: %+v", err)
+	}
+}
+
+// download an image.
+func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error) {
+	log.Printf("[DEBUG] downloading image %s", imgURL)
+
+	timeout := 60 * time.Second // default
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	client := http.Client{Timeout: 30 * time.Second}
+	var resp *http.Response
+	err := repeater.NewDefault(5, time.Second).Do(ctx, func() error {
+		var e error
+		req, e := http.NewRequest("GET", imgURL, nil)
+		if e != nil {
+			return errors.Wrapf(e, "failed to make request for %s", imgURL)
+		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't download image %s", imgURL)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
+	}
+
+	imgData, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("unable to read image body")
+	}
+	return imgData, nil
+}

backend/app/rest/proxy/image_test.go

@@ -3,19 +3,50 @@ package proxy
 import (
 	"encoding/base64"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
-func TestPicture_Extract(t *testing.T) {
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestImage_Extract(t *testing.T) {
 	tbl := []struct {
 		inp string
 		res []string
@@ -45,94 +76,228 @@ func TestPicture_Extract(t *testing.T) {
 			[]string{},
 		},
 	}
-	img := Image{Enabled: true}
+	img := Image{HTTP2HTTPS: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			res, err := img.extract(tt.inp, func(src string) bool { return strings.HasPrefix(src, "http://") })
+			assert.NoError(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
-func TestPicture_Replace(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+func TestImage_Replace(t *testing.T) {
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
 		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 }
 
 func TestImage_Routes(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
-	httpSrv := imgHTTPServer(t)
-	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Times(2).Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	t.Logf("%+v", resp.Header)
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
 	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 
 	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
 	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+}
+
+func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+}
+
+func TestImage_RoutesCachingImage(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{MaxSize: 1500}),
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
+
+	imgURL := httpSrv.URL + "/image/img1.png"
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
+
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+	imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
+	imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
+}
+
+func TestImage_RoutesUsingCachedImage(t *testing.T) {
+	imageStore := image.MockStore{}
+	img := Image{
+		CacheExternal: true,
+		RemarkURL:     "https://demo.remark42.com",
+		RoutePath:     "/api/v1/proxy",
+		ImageService:  image.NewService(&imageStore, image.ServiceParams{}),
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+
+	// In order to validate that cached data used cache "will return" some other data from what http server would
+	testImage := []byte(fmt.Sprintf("%256s", "X"))
+	imageStore.On("Load", mock.Anything).Once().Return(testImage, nil)
+
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "256", resp.Header["Content-Length"][0])
+	assert.Equal(t, "text/plain; charset=utf-8", resp.Header["Content-Type"][0],
+		"if you save text you receive text/plain in response, that's only fair option you got")
+
+	imageStore.AssertCalled(t, "Load", mock.Anything)
 }
 
 func TestImage_RoutesTimedOut(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
-	router := img.Routes()
+	imageStore := image.MockStore{}
+	img := Image{
+		HTTP2HTTPS:   true,
+		RemarkURL:    "https://demo.remark42.com",
+		RoutePath:    "/api/v1/proxy",
+		Timeout:      50 * time.Millisecond,
+		ImageService: image.NewService(&imageStore, image.ServiceParams{}),
+	}
 
-	httpSrv := imgHTTPServer(t)
-	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
 	defer ts.Close()
+	httpSrv := imgHTTPTestsServer(t)
+	defer httpSrv.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+
+	// no image supposed to be cached
+	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
+
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	assert.NoError(t, resp.Body.Close())
+	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
 }
 
-func TestPicture_Convert(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
+func TestImage_ConvertProxyMode(t *testing.T) {
+	img := Image{HTTP2HTTPS: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
 	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
 	assert.Equal(t, `<img src="https://radio-t.com/img3.png"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
 
-	img = Image{Enabled: true, RoutePath: "/img", RemarkURL: "http://example.com"}
+	img = Image{HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "http://example.com"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "http:// remark url, no proxy")
 
-	img = Image{Enabled: false, RoutePath: "/img"}
+	img = Image{HTTP2HTTPS: false, RoutePath: "/img"}
 	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
 	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
 }
 
-func imgHTTPServer(t *testing.T) *httptest.Server {
+func TestImage_ConvertCachingMode(t *testing.T) {
+	img := Image{CacheExternal: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+
+	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="https://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cHM6Ly9yYWRpby10LmNvbS9pbWczLnBuZw=="/> xyz <img src="https://remark42.com/img?src=aHR0cHM6Ly9pbWFnZXMucGV4ZWxzLmNvbS82NzYzNi9pbWc0LmpwZWc=">`, r)
+
+	r = img.Convert(`<img src="https://remark42.com/pictures/1.png"/>`)
+	assert.Equal(t, `<img src="https://remark42.com/pictures/1.png"/>`, r)
+
+	img = Image{CacheExternal: false, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/>`)
+	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/>`, r)
+
+	// both Caching and Proxy enabled
+	img = Image{CacheExternal: true, HTTP2HTTPS: true, RoutePath: "/img", RemarkURL: "https://remark42.com"}
+	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
+	assert.Equal(t, `<img src="https://remark42.com/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="https://remark42.com/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
+}
+
+func imgHTTPTestsServer(t *testing.T) *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/image/img1.png" {
 			t.Log("http img request", r.URL)
-			w.Header().Add("Content-Length", "123")
+			w.Header().Add("Content-Length", "1462")
 			w.Header().Add("Content-Type", "image/png")
-			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write(gopherPNGBytes())
 			assert.NoError(t, err)
 			return
 		}

backend/app/rest/user.go

@@ -35,17 +35,27 @@ func GetUserInfo(r *http.Request) (user store.User, err error) {
 		Admin:    u.IsAdmin(),
 		Verified: u.BoolAttr("verified"),
 		Blocked:  u.BoolAttr("blocked"),
+		SiteID:   u.Audience,
 	}, nil
+}
 
+// GetUserOrEmpty attempts to get user info from request and returns empty object if failed
+func GetUserOrEmpty(r *http.Request) store.User {
+	user, err := GetUserInfo(r)
+	if err != nil {
+		return store.User{}
+	}
+	return user
 }
 
 // SetUserInfo sets user into request context
 func SetUserInfo(r *http.Request, user store.User) *http.Request {
 	u := token.User{
-		ID:      user.ID,
-		Name:    user.Name,
-		Picture: user.Picture,
-		IP:      user.IP,
+		ID:       user.ID,
+		Name:     user.Name,
+		Picture:  user.Picture,
+		IP:       user.IP,
+		Audience: user.SiteID,
 		Attributes: map[string]interface{}{
 			"blocked":  user.Blocked,
 			"verified": user.Verified,

backend/app/rest/user_test.go

@@ -5,22 +5,23 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestUser_GetUserInfo(t *testing.T) {
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
+	assert.Error(t, err, "no user info")
 
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
+	r = SetUserInfo(r, store.User{Name: "test", ID: "id", SiteID: "test"})
 	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "test", ID: "id", SiteID: "test"}, u)
 }
 
-func TestUSer_MustGetUserInfo(t *testing.T) {
+func TestUser_MustGetUserInfo(t *testing.T) {
 	defer func() {
 		if r := recover(); r != nil {
 			t.Log("recovered from panic")
@@ -28,12 +29,12 @@ func TestUSer_MustGetUserInfo(t *testing.T) {
 	}()
 
 	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	_ = MustGetUserInfo(r)
 	assert.Fail(t, "should panic")
 
 	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
 	u := MustGetUserInfo(r)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
 }

backend/app/store/admin/admin.go

@@ -3,6 +3,7 @@ package admin
 
 import (
 	"errors"
+	"strings"
 
 	log "github.com/go-pkgz/lgr"
 )
@@ -10,29 +11,35 @@ import (
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key() (key string, err error)
-	Admins(siteID string) (ids []string)
-	Email(siteID string) (email string)
+	Admins(siteID string) (ids []string, err error)
+	Email(siteID string) (email string, err error)
+	Enabled(siteID string) (ok bool, err error)
+	OnEvent(siteID string, et EventType) error
 }
 
-// StaticStore implements keys.Store with a single, predefined key
+// EventType indicates type of the event
+type EventType int
+
+// enum of all event types
+const (
+	EvCreate EventType = iota
+	EvDelete
+	EvUpdate
+	EvVote
+)
+
+// StaticStore implements keys.Store with a single set of admins and email for all sites
 type StaticStore struct {
 	admins []string
 	email  string
 	key    string
-}
-
-// Key returns static key for all sites, allows empty site
-func (s *StaticStore) Key() (key string, err error) {
-	if s.key == "" {
-		return "", errors.New("empty key for static key store")
-	}
-	return s.key, nil
+	sites  []string
 }
 
 // NewStaticStore makes StaticStore instance with given key
-func NewStaticStore(key string, admins []string, email string) *StaticStore {
+func NewStaticStore(key string, sites []string, admins []string, email string) *StaticStore {
 	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
-	return &StaticStore{key: key, admins: admins, email: email}
+	return &StaticStore{key: key, sites: sites, admins: admins, email: email}
 }
 
 // NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
@@ -40,12 +47,36 @@ func NewStaticKeyStore(key string) *StaticStore {
 	return &StaticStore{key: key, admins: []string{}, email: ""}
 }
 
+// Key returns static key, same for all sites
+func (s *StaticStore) Key() (key string, err error) {
+	if s.key == "" {
+		return "", errors.New("empty key for static key store")
+	}
+	return s.key, nil
+}
+
 // Admins returns static list of admin's ids, the same for all sites
-func (s *StaticStore) Admins(string) (ids []string) {
-	return s.admins
+func (s *StaticStore) Admins(string) (ids []string, err error) {
+	return s.admins, nil
 }
 
 // Email gets static email address
-func (s *StaticStore) Email(string) (email string) {
-	return s.email
+func (s *StaticStore) Email(string) (email string, err error) {
+	return s.email, nil
 }
+
+// Enabled if always true for StaticStore
+func (s *StaticStore) Enabled(site string) (ok bool, err error) {
+	if len(s.sites) == 0 {
+		return true, nil
+	}
+	for _, allowedSite := range s.sites {
+		if strings.EqualFold(allowedSite, site) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// OnEvent doesn nothing for StaticStore
+func (s *StaticStore) OnEvent(_ string, _ EventType) error { return nil }

backend/app/store/admin/admin_test.go

@@ -3,61 +3,29 @@ package admin
 import (
 	"testing"
 
-	"github.com/globalsign/mgo"
-	"github.com/go-pkgz/mongo"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestStaticStore_Get(t *testing.T) {
-	var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com")
+	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"}, []string{"123", "xyz"}, "aa@example.com")
 
 	k, err := ks.Key()
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 
-	a := ks.Admins("any")
+	a, err := ks.Admins("s1")
+	assert.NoError(t, err)
 	assert.Equal(t, []string{"123", "xyz"}, a)
 
-	email := ks.Email("blah")
+	email, err := ks.Email("s2")
+	assert.NoError(t, err)
 	assert.Equal(t, "aa@example.com", email)
-}
 
-func TestMongoStore_Get(t *testing.T) {
-	conn, err := mongo.MakeTestConnection(t)
-	require.NoError(t, err)
-	var ms Store = NewMongoStore(conn, "secret")
-
-	recs := []mongoRec{
-		{"site1", []string{"i11", "i12"}, "e1"},
-		{"site2", []string{"i21", "i22"}, "e2"},
-	}
-	err = conn.WithCollection(func(coll *mgo.Collection) error {
-		if e1 := coll.Insert(recs[0]); e1 != nil {
-			return e1
-		}
-		return coll.Insert(recs[1])
-	})
-	require.NoError(t, err)
-
-	admins := ms.Admins("site1")
-	assert.Equal(t, []string{"i11", "i12"}, admins)
-	email := ms.Email("site1")
-	assert.Equal(t, "e1", email)
-	key, err := ms.Key()
+	enabled, err := ks.Enabled("s3")
 	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
+	assert.Equal(t, true, enabled)
 
-	admins = ms.Admins("site2")
-	assert.Equal(t, []string{"i21", "i22"}, admins)
-	email = ms.Email("site2")
-	assert.Equal(t, "e2", email)
-	key, err = ms.Key()
+	enabled, err = ks.Enabled("serr")
 	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("no-site-in-db")
-	assert.Equal(t, []string{}, admins)
-	email = ms.Email("no-site-in-db")
-	assert.Equal(t, "", email)
+	assert.Equal(t, false, enabled)
 }

backend/app/store/admin/mongo.go

@@ -1,56 +0,0 @@
-package admin
-
-import (
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	log "github.com/go-pkgz/lgr"
-
-	"github.com/go-pkgz/mongo"
-)
-
-// MongoStore implements admin.Store with mongo backend
-type MongoStore struct {
-	connection *mongo.Connection
-	key        string
-}
-
-type mongoRec struct {
-	SiteID string   `bson:"site"`
-	IDs    []string `bson:"admin_ids"`
-	Email  string   `bson:"admin_email"`
-}
-
-// NewMongoStore makes admin Store for mongo's connection
-func NewMongoStore(conn *mongo.Connection, key string) *MongoStore {
-	log.Printf("[DEBUG] make mongo admin store with %+v", conn)
-	return &MongoStore{connection: conn, key: key}
-}
-
-// Key executes find by siteID and returns substructure with secret key
-func (m *MongoStore) Key() (key string, err error) {
-	return m.key, nil
-}
-
-// Admins executes find by siteID and returns admins ids
-func (m *MongoStore) Admins(siteID string) (ids []string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return []string{}
-	}
-	return resp.IDs
-}
-
-// Email executes find by siteID and returns admin's email
-func (m *MongoStore) Email(siteID string) (email string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return ""
-	}
-	return resp.Email
-}

backend/app/store/admin/remote.go

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	jrpc.Client
+}
+
+// Key returns the key, same for all sites
+func (r *RPC) Key() (key string, err error) {
+	resp, err := r.Call("admin.key")
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &key)
+	return key, err
+}
+
+// Admins returns list of admin's ids for given site
+func (r *RPC) Admins(siteID string) (ids []string, err error) {
+	resp, err := r.Call("admin.admins", siteID)
+	if err != nil {
+		return []string{}, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+		return []string{}, err
+	}
+	return ids, nil
+}
+
+// Email gets email address for given site
+func (r *RPC) Email(siteID string) (email string, err error) {
+	resp, err := r.Call("admin.email", siteID)
+	if err != nil {
+		return "", err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &email); err != nil {
+		return "", err
+	}
+	return email, nil
+}
+
+// Enabled returns true if allowed
+func (r *RPC) Enabled(siteID string) (ok bool, err error) {
+	resp, err := r.Call("admin.enabled", siteID)
+	if err != nil {
+		return false, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ok); err != nil {
+		return false, err
+	}
+	return ok, nil
+}
+
+// OnEvent reacts (register) events about data modification
+func (r *RPC) OnEvent(siteID string, et EventType) error {
+	_, err := r.Call("admin.event", siteID, et)
+	if err != nil {
+		return err
+	}
+	return nil
+}

backend/app/store/admin/remote_test.go

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemote_Key(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.key","id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Admins(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.admins","params":"site-1","id":1}`,
+		`{"result":["id1","id2"],"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Admins("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Email(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.email","params":"site-1","id":1}`,
+		`{"result":"bbb@example.com","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Email("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, "bbb@example.com", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Enables(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.enabled","params":"site-1","id":1}`,
+		`{"result":true,"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Enabled("site-1")
+	assert.NoError(t, err)
+	assert.True(t, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_OnEvent(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.event","params":["site-1",2],"id":1}`, `{"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.OnEvent("site-1", EvUpdate)
+	assert.NoError(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		_, _ = fmt.Fprint(w, resp)
+	}))
+}

backend/app/store/comment.go

@@ -1,8 +1,10 @@
 package store
 
 import (
+	"fmt"
 	"html/template"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/microcosm-cc/bluemonday"
@@ -10,21 +12,22 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID          string          `json:"id" bson:"_id"`
-	ParentID    string          `json:"pid"`
-	Text        string          `json:"text"`
-	Orig        string          `json:"orig,omitempty"`
-	User        User            `json:"user"`
-	Locator     Locator         `json:"locator"`
-	Score       int             `json:"score"`
-	Votes       map[string]bool `json:"votes,omitempty"`
-	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
-	Controversy float64         `json:"controversy,omitempty"`
-	Timestamp   time.Time       `json:"time" bson:"time"`
-	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
-	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
-	PostTitle   string          `json:"title,omitempty" bson:"title"`
+	ID          string                 `json:"id" bson:"_id"`
+	ParentID    string                 `json:"pid"`
+	Text        string                 `json:"text"`
+	Orig        string                 `json:"orig,omitempty"`
+	User        User                   `json:"user"`
+	Locator     Locator                `json:"locator"`
+	Score       int                    `json:"score"`
+	Votes       map[string]bool        `json:"votes,omitempty"`
+	VotedIPs    map[string]VotedIPInfo `json:"voted_ips,omitempty"` // voted ips (hashes) with TS
+	Vote        int                    `json:"vote"`                // vote for the current user, -1/1/0.
+	Controversy float64                `json:"controversy,omitempty"`
+	Timestamp   time.Time              `json:"time" bson:"time"`
+	Edit        *Edit                  `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Pin         bool                   `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool                   `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string                 `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post
@@ -55,6 +58,12 @@ type BlockedUser struct {
 	Until time.Time `json:"time"`
 }
 
+// VotedIPInfo keeps timestamp and voting value (direction). Used as VotedIPs value
+type VotedIPInfo struct {
+	Timestamp time.Time
+	Value     bool
+}
+
 // DeleteMode defines how much comment info will be erased
 type DeleteMode int
 
@@ -66,6 +75,7 @@ const (
 
 // Maximum length for URL text shortening.
 const shortURLLen = 48
+const snippetLen = 200
 
 // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
 // autogen fields and reset everything users not supposed to provide
@@ -100,10 +110,60 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 // Sanitize clean dangerous html/js from the comment
 func (c *Comment) Sanitize() {
 	p := bluemonday.UGCPolicy()
-	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
+	p.AllowAttrs("class").Matching(regexp.MustCompile("^chroma$")).OnElements("pre")
+	// this is list of <span> tag classes which could be produced by chroma code renderer
+	// source: https://github.com/alecthomas/chroma/blob/022b6f4fc2c4aa819aac18363c8de3f70619200b/types.go#L221-L316
+	const codeSpanClassRegex = "^(chroma|ln|lnt|hl|lntable|lntd|w|err|x|esc|k|kc" +
+		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
+		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
+		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
+	p.AllowAttrs("class").Matching(regexp.MustCompile(codeSpanClassRegex)).OnElements("span")
 	c.Text = p.Sanitize(c.Text)
 	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = template.HTMLEscapeString(c.User.Name)
-	c.User.Picture = p.Sanitize(c.User.Picture)
+	c.User.Name = c.escapeHTMLWithSome(c.User.Name)
+	c.User.Picture = c.SanitizeAsURL(c.User.Picture)
+	c.Locator.URL = c.SanitizeAsURL(c.Locator.URL)
+}
+
+// Snippet from comment's text
+func (c *Comment) Snippet(limit int) string {
+	if limit <= 0 {
+		limit = snippetLen
+	}
+	cleanText := strings.Replace(c.Text, "\n", " ", -1)
+	size := len([]rune(cleanText))
+	if size < limit {
+		return cleanText
+	}
+	snippet := []rune(cleanText)[:size]
+	// go back in snippet and found the first space
+	for i := len(snippet) - 1; i >= 0; i-- {
+		if snippet[i] == ' ' {
+			snippet = snippet[:i]
+			break
+		}
+	}
+	return string(snippet) + " ..."
+}
+
+var reHref = regexp.MustCompile(`<a\s+(?:[^>]*?\s+)?href="([^"]*)"`)
+
+// SanitizeAsURL drops dangerous code from a url.
+// It wraps input with href to trigger bluemonday sanitizer and cleans href after sanitizing done
+func (c *Comment) SanitizeAsURL(inp string) string {
+	h := fmt.Sprintf(`<a href="%s">`, inp)
+	clean := bluemonday.UGCPolicy().Sanitize(h)
+	if match := reHref.FindStringSubmatch(clean); len(match) > 1 {
+		return match[1]
+	}
+	return "" // this shouldn't happen as we build the href
+}
+
+func (c *Comment) escapeHTMLWithSome(inp string) string {
+	res := template.HTMLEscapeString(inp)
+	res = strings.Replace(res, "&#34;", "\"", -1)
+	res = strings.Replace(res, "&#39;", "'", -1)
+	res = strings.Replace(res, "&amp;", "&", -1)
+	return res
 }

backend/app/store/comment_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -27,13 +28,44 @@ func TestComment_Sanitize(t *testing.T) {
 		{
 			inp: Comment{
 				Text: "blah 123" + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 			out: Comment{
 				Text: `blah 123` + "\n\t",
-				User: User{ID: "id", Name: "xyz"},
+				User: User{ID: "id", Name: "xyz-123"},
 			},
 		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123", User: User{Name: "name <> & ' ` \""}},
+			out: Comment{Text: `blah &amp; &amp; 123`, User: User{Name: "name &lt;&gt; & ' ` \""}},
+		},
+
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert('XSS1')"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "javascript:alert(document.domain)//"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: ""}},
+		},
+		{
+			inp: Comment{Text: "blah blah", Locator: Locator{URL: "<script>alert()</script>"}},
+			out: Comment{Text: "blah blah", Locator: Locator{URL: "%3Cscript%3Ealert%28%29%3C/script%3E"}},
+		},
+		{
+			inp: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+			out: Comment{Text: "blah blah",
+				Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -120,3 +152,63 @@ func TestComment_SetDeletedHard(t *testing.T) {
 	assert.False(t, comment.Pin)
 	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
 }
+
+func TestComment_Snippet(t *testing.T) {
+	tbl := []struct {
+		limit int
+		inp   string
+		out   string
+	}{
+		{0, "", ""},
+		{-1, "test\nblah", "test blah"},
+		{5, "test\nblah", "test ..."},
+		{5, "xyz12345 xxx", "xyz12345 ..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			c := Comment{Text: tt.inp}
+			out := c.Snippet(tt.limit)
+			assert.Equal(t, tt.out, out)
+		})
+	}
+}
+
+func TestComment_SanitizeAsURL(t *testing.T) {
+
+	tbl := []struct {
+		inp, out string
+	}{
+		{
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+			"https://radio-t.com/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085",
+		},
+		{
+			"javascript:alert(document.domain)//",
+			"",
+		},
+		{
+			"<script>alert()</script>",
+			"%3Cscript%3Ealert%28%29%3C/script%3E",
+		},
+		{
+			"<a href=javascript:alert(document.domain)//>xxx</a>",
+			"",
+		},
+	}
+
+	for i, tt := range tbl {
+		tt := tt
+		c := Comment{}
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.out, c.SanitizeAsURL(tt.inp))
+		})
+	}
+
+}