clean up comment HasReplies cache on child comment deletion

Previously, the cache kept the entry and deletion of the parent comment
after child deletion was not possible for the rest
of cache life (5m) duration. Now it's possible to delete
a parent comment after the deletion of the child comment
by a user or admin.

Resolves #1481

.dockerignore

@@ -1,14 +1,27 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
-/web/public/
 /.vscode/
 /.idea/
-#/.git/
+/bin/
+/.git/
+
+# frontend files not needed in docker image
+/frontend/node_modules/
+/frontend/apps/remark42/node_modules/
+/frontend/apps/remark42/public/
+# e2e tests arficats
+/frontend/e2e/playwright-report/
+/frontend/e2e/playwright/.cache/
+/frontend/e2e/test-results/
 
 # source files
 docker-compose.yml
+compose-dev-backend.yml
+compose-dev-frontend.yml
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
 rest-client.env.json
 
 # generated files
@@ -19,5 +32,6 @@ debug
 debug.test
 *.prof
 *.test
-/bin/
 remark42
+/backend/var/
+/playwright-report/

.drone.yml

@@ -1,102 +0,0 @@
-workspace:
-  base: /go
-  path: src/github.com/umputun/remark
-
-pipeline:
-  
-  build:
-    image: golang:1.10-alpine
-    commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
-      - cd backend/app
-      - go build -v ./...
-
-  docker_master:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
-    when:
-      branch: [master, release/*]
-      event: push
-
-  docker_tag:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-    tags: 
-      - ${DRONE_TAG}
-      - latest
-    when:
-      event: tag
-
-  artifacts_tag:
-    image: plugins/docker
-    dockerfile: Dockerfile.artifacts
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
-    when:
-      event: tag
-
-  docker_branch:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE=${DRONE}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-      - DRONE_BRANCH=${DRONE_BRANCH}
-    tags:
-      - ${DRONE_COMMIT_BRANCH/\//-}
-    when:
-      branch:
-        exclude: [master, release/*]
-        event: push
-
-  deploy:
-    image: appleboy/drone-ssh
-    host: remark42.com
-    username: umputun
-    port: 22
-    secrets: [ ssh_key ]
-    script:
-      - cd /srv/remark
-      - docker-compose pull
-      - docker-compose up -d
-    when:
-      branch: master
-      event: push
-
-  notify:
-    image: drillster/drone-email
-    host: smtp.mailgun.org
-    port: 25
-    username: email_username
-    password: email_password
-    from: drone@mg.umputun.com
-    recipients: [ umputun@gmail.com ]
-    secrets: [ email_username, email_password ]
-    when:
-      status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]
-

.editorconfig

@@ -0,0 +1,11 @@
+[*]
+indent_style = tab
+insert_final_newline = true
+
+[*.md]
+indent_style = space
+trim_trailing_whitespace = false
+
+[*.{yml,json}]
+indent_size = 2
+indent_style = space

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*             @umputun
+frontend/*    @akellbl4 @Mavrin

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+patreon: remark42

.github/workflows/ci-backend.yml

@@ -0,0 +1,65 @@
+name: backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - ".github/workflows/ci-test-backend.yml"
+      - "backend/**"
+      - "!backend/scripts/**"
+      - "!**.md"
+  pull_request:
+    types: [opened, reopened]
+    paths:
+      - ".github/workflows/ci-test-backend.yml"
+      - "backend/**"
+      - "!backend/scripts/**"
+      - "!**.md"
+jobs:
+  test:
+    name: Test & Coverage
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: debug if needed
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GITHUB_WORKSPACE v1.49.0
+          go install github.com/mattn/goveralls@latest
+
+      - name: test and lint backend
+        run: |
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          TZ: "America/Chicago"
+
+      - name: test and lint examples
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
+
+      - name: submit coverage
+        run: goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-build.yml

@@ -0,0 +1,102 @@
+name: build
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+    paths:
+      - ".github/workflows/ci-build.yml"
+      - "backend/**"
+      - "frontend/apps/**"
+      - ".dockerignore"
+      - "docker-init.sh"
+      - "Dockerfile"
+      - "!**.md"
+      - "!frontend/packages/**"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-build.yml"
+      - "backend/**"
+      - "frontend/apps/**"
+      - ".dockerignore"
+      - "docker-init.sh"
+      - "Dockerfile"
+      - "!**.md"
+
+jobs:
+  build-images:
+    name: Build Docker images
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: build docker image without pushing (only outside master)
+        if: ${{ github.ref != 'refs/heads/master' }}
+        run: |
+          docker buildx build \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
+              --platform linux/amd64 .
+
+      - name: build example docker image without pushing (only outside master)
+        if: ${{ github.ref != 'refs/heads/master' }}
+        run: |
+          docker buildx build \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
+              --platform linux/amd64 -f backend/_example/memory_store/Dockerfile .
+
+      - name: build and deploy master image to ghcr.io and dockerhub
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo "GITHUB_REF=${GITHUB_REF}, GITHUB_SHA=${GITHUB_SHA}, GIT_BRANCH=${ref}"
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u umputun --password-stdin
+          docker buildx build --push \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
+              --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} --build-arg GITHUB_REF=${GITHUB_REF} \
+              --platform linux/amd64,linux/arm/v7,linux/arm64 \
+              -t ghcr.io/umputun/remark42:${ref} -t umputun/remark42:${ref} .
+
+      - name: deploy tagged (latest) to ghcr.io and dockerhub
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo "GITHUB_REF=${GITHUB_REF}, GITHUB_SHA=${GITHUB_SHA}, GIT_BRANCH=${ref}"
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u umputun --password-stdin
+          docker buildx build --push \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
+              --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} --build-arg GITHUB_REF=${GITHUB_REF} \
+              --platform linux/amd64,linux/arm/v7,linux/arm64 \
+              -t ghcr.io/umputun/remark42:${ref} -t ghcr.io/umputun/remark42:latest \
+              -t umputun/remark42:${ref} -t umputun/remark42:latest .
+
+      - name: remote deployment to remark42.com from master
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
+        run: curl -s https://jess.umputun.com/update/remark42-core/${UPDATER_KEY}

.github/workflows/ci-frontend-api.yml

@@ -0,0 +1,153 @@
+name: "@remark42/api"
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/ci-frontend-api.yml"
+      - "frontend/packages/**"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-frontend-api.yml"
+      - "frontend/packages/**"
+      - "!**.md"
+
+jobs:
+  type-check:
+    name: Type check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run type check
+        run: pnpm type-check:api
+        working-directory: ./frontend
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run linters
+        run: pnpm lint:api
+        working-directory: ./frontend/
+
+  test:
+    name: Tests & Coverage
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Test & Coverage
+        run: pnpm coverage:api
+        working-directory: ./frontend
+
+      - name: Submit coverage
+        run: ${{ github.workspace }}/frontend/apps/remark42/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-frontend.yml

@@ -0,0 +1,220 @@
+name: frontend
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/ci-frontend.yml"
+      - "frontend/apps/remark42/**"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-frontend.yml"
+      - "frontend/apps/remark42/**"
+      - "!**.md"
+
+jobs:
+  translations-check:
+    name: Translations check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Translations check
+        run: pnpm translation-check
+        working-directory: ./frontend/apps/remark42
+
+  type-check:
+    name: Type check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run type check
+        run: pnpm type-check
+        working-directory: ./frontend/apps/remark42
+
+  lint:
+    name: Eslint & Stylelint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run linters
+        run: pnpm lint
+        working-directory: ./frontend/apps/remark42
+
+  size-limit:
+    name: Size limit
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Check bundle size
+        uses: andresz1/size-limit-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          directory: ./frontend/apps/remark42
+
+  test:
+    name: Tests & Coverage
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.15.1]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.0.1
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Test & Coverage
+        run: pnpm coverage
+        working-directory: ./frontend/apps/remark42
+
+      - name: Submit coverage
+        run: ${{ github.workspace }}/frontend/apps/remark42/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-site.yml

@@ -0,0 +1,69 @@
+name: site
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+    paths:
+      - ".github/workflows/ci-site.yml"
+      - "site/**"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-site.yml"
+      - "site/**"
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: build and deploy master image to ghcr.io and dockerhub
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        working-directory: ./site
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo GITHUB_REF - $ref
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
+            -t ghcr.io/umputun/remark42-site:${ref} .
+
+      - name: deploy tagged (latest) to ghcr.io and dockerhub
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        working-directory: ./site
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo "GITHUB_REF=$ref, GITHUB_SHA=${GITHUB_SHA}"
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
+            -t ghcr.io/umputun/remark42-site:${ref} -t ghcr.io/umputun/remark42-site:latest .
+
+      - name: remote site deployment from master
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
+        run: curl https://jess.umputun.com/update/remark42-site/${UPDATER_KEY}

.github/workflows/e2e-tests.yml

@@ -0,0 +1,34 @@
+name: e2e
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - "frontend/apps/remark42/**"
+      - "frontend/e2e/**"
+
+  pull_request:
+    branches: [master]
+    paths:
+      - "frontend/apps/remark42/**"
+      - "frontend/e2e/**"
+
+jobs:
+  tests:
+    name: Tests
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build & run containers
+        id: tests
+        run: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: playwright-report
+          path: ./playwright-report/
+          retention-days: 30

.gitignore

@@ -8,13 +8,20 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
-/web/public/
-/web/coverage
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
 .mongo
 remark42
-/bin/
+/bin/
+/backend/var/
+/backend/app/var/
+/backend/app/cmd/web/
+/backend/*.html.tmpl
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor
+http-client.env.json
+/playwright-report/

.travis.yml

@@ -1,25 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
-      .
-  - docker rm -f mongo
-  

Dockerfile

@@ -1,107 +1,115 @@
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM --platform=$BUILDPLATFORM node:16.15.1-alpine AS frontend-deps
+
+ARG SKIP_FRONTEND_TEST
+ARG SKIP_FRONTEND_BUILD
+
+WORKDIR /srv/frontend/
+
+COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
+COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/
+
+RUN \
+  if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
+    apk add --no-cache --update git && \
+    npm i -g pnpm; \
+  fi
+
+RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 \
+  if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
+    pnpm i; \
+  fi
+
+
+FROM --platform=$BUILDPLATFORM frontend-deps AS build-frontend
+
+ARG SKIP_FRONTEND_TEST
+ARG SKIP_FRONTEND_BUILD
+ENV CI=true
+
+WORKDIR /srv/frontend/apps/remark42/
+
+COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
+
+RUN \
+  if [ -z "$SKIP_FRONTEND_TEST" ]; then \
+    pnpm lint type-check translation-check test; \
+  else \
+    echo 'Skip frontend test'; \
+  fi
+
+RUN \
+  if [ -z "$SKIP_FRONTEND_BUILD" ]; then \
+    pnpm build; \
+  else \
+    mkdir /srv/frontend/apps/remark42/public; \
+    echo 'Skip frontend build'; \
+  fi
+
+FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
-ARG DRONE
-ARG DRONE_TAG
-ARG DRONE_COMMIT
-ARG DRONE_BRANCH
-ARG DRONE_PULL_REQUEST
-
+ARG GITHUB_REF
+ARG GITHUB_SHA
+ARG GIT_BRANCH
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
+ARG BACKEND_TEST_TIMEOUT
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD .git /go/src/github.com/umputun/remark/.git
+ADD backend /build/backend
+# to embed the frontend files statically into Remark42 binary
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
+RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
+WORKDIR /build/backend
+
+# install gcc in order to be able to go test package with -race
+RUN apk --no-cache add gcc libc-dev
+
+RUN echo go version: `go version`
 
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
-    else echo "skip backend test" ; fi
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        golangci-lint run --config ../.golangci.yml ./... ; \
+    else \
+      echo "skip backend tests and linter" \
+    ; fi
 
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
-# linters
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
-    else echo "skip backend linters" ; fi
-
-# coverage report
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    mkdir -p target && /script/coverage.sh ; \
-    else echo "skip backend coverage" ; fi
-
-# submit coverage to coverals if COVERALLS_TOKEN in env
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
-
-# if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
-    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    version="$(/script/version.sh)" && \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
+FROM umputun/baseimage:app-v1.9.2
 
-FROM node:10.11-alpine as build-frontend-deps
+ARG GITHUB_SHA
 
-ARG CI
-
-RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
-
-FROM node:10.11-alpine as build-frontend
-
-ARG CI
-ARG SKIP_FRONTEND_TEST
-ARG NODE_ENV=production
-
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
-    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
-    else echo "skip frontend tests and lint" ; npm run build ; fi && \
-    rm -rf ./node_modules
-
-
-FROM umputun/baseimage:app-latest
+LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
+      org.opencontainers.image.description="Remark42 comment engine" \
+      org.opencontainers.image.documentation="https://remark42.com/docs/getting-started/" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.title="Remark42" \
+      org.opencontainers.image.url="https://remark42.com/" \
+      org.opencontainers.image.revision="${GITHUB_SHA}"
 
 WORKDIR /srv
 
-ADD entrypoint.sh /entrypoint.sh
+ADD docker-init.sh /entrypoint.sh
 ADD backend/scripts/backup.sh /usr/local/bin/backup
 ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /srv/web/
+COPY docker-init.sh /srv/init.sh
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["server"]
-ENTRYPOINT ["/entrypoint.sh"]
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -1,96 +1,64 @@
-FROM node:10.11-alpine as build-frontend-deps
+FROM node:16.15.1-alpine AS frontend-deps
 
-ARG CI
-ARG DRONE
-ARG DRONE_TAG
-ARG DRONE_COMMIT
-ARG DRONE_BRANCH
+ENV CI=true
 
-ENV SKIP_FRONTEND_TEST=true
+WORKDIR /srv/frontend
 
-RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
+COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/package.json
 
-FROM node:10.11-alpine as build-frontend
+RUN apk add --no-cache --update git && npm i -g pnpm
+RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 pnpm i
 
-ARG CI
-ARG NODE_ENV=production
-ENV SKIP_FRONTEND_TEST=true
+FROM frontend-deps AS build-frontend
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
-    npm run build && \
-    rm -rf ./node_modules
+ENV NODE_ENV=production
+ENV CI=true
 
+WORKDIR /srv/frontend/apps/remark42/
+COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
+RUN pnpm build
 
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
 
 ARG GITHUB_TOKEN
-ENV SKIP_BACKEND_TEST=true
+ARG GITHUB_REF
+ARG GITHUB_SHA
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD README.md /go/src/github.com/umputun/remark/
-ADD LICENSE /go/src/github.com/umputun/remark/
-COPY --from=build-frontend /srv/web/public/ web
+WORKDIR /build/backend
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
+
+RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
 
 RUN \
-    export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
-    sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
-    sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
-    go get -v github.com/rakyll/statik && \
-    statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
-    ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
-    ls -la /go/src/github.com/umputun/remark/backend/web/
-
-# if DRONE presented use DRONE_* git env to make version
-RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
-    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
-    echo "version=$version" && \
+  version=$("/script/version.sh") && echo "version=${version}" && \
     GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
-    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=darwin GOARCH=arm64 go build -o remark42.darwin-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
 
 RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "runs outside of drone" && tag=""; \
-    else tag=_${DRONE_TAG}; fi && \
     apk add --no-cache --update zip && \
-    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
-    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
-    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
+    tar cvzf remark42.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
+    tar cvzf remark42.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
+    tar cvzf remark42.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
+    tar cvzf remark42.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42.darwin-arm64.tar.gz remark42.darwin-arm64 LICENSE README.md && \
+    tar cvzf remark42.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
+    zip remark42.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
-# upload to github
-RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "skip upload to github" ; \
-    else \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
 RUN ls -la /artifacts/*
 CMD ["sleep", "100"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2021 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,5 +1,7 @@
 OS=linux
 ARCH=amd64
+GITHUB_REF=$(shell git rev-parse --symbolic-full-name HEAD)
+GITHUB_SHA=$(shell git rev-parse --short HEAD)
 
 bin:
 	docker build -f Dockerfile.artifacts -t remark42.bin .
@@ -9,10 +11,18 @@ bin:
 	docker rm -f remark42.bin
 
 docker:
-	docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
+	DOCKER_BUILDKIT=1 docker build -t umputun/remark42 --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) \
+		--build-arg CI=true --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
 
-deploy:
-	docker build -f Dockerfile.artifacts -t remark42.bin .
+dockerx:
+	docker buildx build --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) --build-arg CI=true \
+		--build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
+		--progress=plain --platform linux/amd64,linux/arm/v7,linux/arm64 \
+		-t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
+
+release:
+	docker build -f Dockerfile.artifacts --no-cache --pull --build-arg CI=true \
+		--build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) -t remark42.bin .
 	- @docker rm -f remark42.bin 2>/dev/null || exit 0
 	- @mkdir -p bin
 	docker run -d --name=remark42.bin remark42.bin
@@ -20,7 +30,23 @@ deploy:
 	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.darwin-arm64.tar.gz bin/remark42.darwin-arm64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
 	docker rm -f remark42.bin
 
-.PHONY: bin
+race_test:
+	cd backend/app && go test -race -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true GITHUB_REF=$(GITHUB_REF) GITHUB_SHA=$(GITHUB_SHA) CI=true \
+		docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,628 +1,49 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# Remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Image Size](https://img.shields.io/docker/image-size/umputun/remark42/master)](https://hub.docker.com/r/umputun/remark42) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://app.codecov.io/gh/umputun/remark42)
 
-Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
+Remark42 is a self-hosted, lightweight and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles, or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, Microsoft, GitHub, Yandex, Patreon and Telegram
+* Login via email
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus and wordpress
-* Markdown support
+* Import from Disqus and WordPress
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
-* Export data to json with automatic backups
+* Telegram, Slack, Webhook and email notifications for Admins (get notified for each new comment)
+* Email and Telegram notifications for users (get notified when someone responds to your comment)
+* Export data to JSON with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
-* Self-contained executable can be deployed directly to Linux, Windows and MacOS
-* Clean, lightweight and fully customizable UI
+* Self-contained executable can be deployed directly to Linux, Windows and macOS
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
-* Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
-* [Privacy focused](#privacy)
+* Integration with automatic SSL (direct and via [nginx-le](https://github.com/nginx-le/nginx-le))
+* [Privacy focused](https://remark42.com/#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
 
-#
+<details><summary>Screenshots</summary>
 
-  - [Install](#install)
-    - [Backend](#backend)
-      - [With Docker](#with-docker) 
-      - [Without docker](#without-docker) 
-      - [Parameters](#parameters)
-        - [Required parameters](#required-parameters)
-      - [Register oauth2 providers](#register-oauth2-providers)
-        - [Google Auth Provider](#google-auth-provider)
-        - [GitHub Auth Provider](#github-auth-provider)
-        - [Facebook Auth Provider](#facebook-auth-provider)
-        - [Yandex Auth Provider](#yandex-auth-provider)
-      - [Initial import from Disqus](#initial-import-from-disqus)
-      - [Initial import from WordPress](#initial-import-from-wordpress)
-      - [Backup and restore](#backup-and-restore)
-        - [Automatic backups](#automatic-backups)
-        - [Manual backup](#manual-backup)
-        - [Restore from backup](#restore-from-backup)
-        - [Backup format](#backup-format)
-      - [Admin users](#admin-users)
-    - [Setup on your website](#setup-on-your-website)
-      - [Comments](#comments)
-      - [Last comments](#last-comments)
-      - [Counter](#counter)
-  - [Build from the source](#build-from-the-source)
-  - [Development](#development)
-    - [Backend development](#backend-development)
-    - [Frontend development](#frontend-development)
-      - [Build](#build)
-      - [Devserver](#devserver)
-  - [API](#api)
-    - [Authorization](#authorization)
-    - [Commenting](#commenting)
-    - [RSS feeds](#rss-feeds)
-    - [Admin](#admin)
-  - [Privacy](#privacy)
-  - [Technical details](#technical-details)
+Comments example:
+![](screenshots/comments.png)
 
+For admin screenshots see [Admin UI documentation](https://remark42.com/docs/manuals/admin-interface/)
+</details>
 
-## Install
+All remark42 documentation is available [by the link](https://remark42.com/docs/getting-started/installation/).
 
-### Backend
+## Contribution
 
-#### With Docker
+In order to start and work on the project locally in development mode check our contribution documentation for [backend](https://remark42.com/docs/contributing/backend/) and [frontend](https://remark42.com/docs/contributing/frontend/).
 
-_this is the recommended way to run remark42_
+If you are interested in adding a new localization please check [these docs](https://remark42.com/docs/contributing/translations/).
 
-* copy provided `docker-compose.yml` and customize for your needs
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
-* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
-* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
+## Related projects
 
-#### Without docker
-
-* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
-* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows) 
-* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
-* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
-
-#### Parameters
-
-| Command line            | Environment             | Default               | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
-
-* command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
-* _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
-* _required_ parameters have to be presented in the environment or provided in command line
-
-##### Required parameters
-
-Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
-
-1. `SECRET` - secret key, can be any long and hard-to-guess string.
-2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
-3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
-
-The minimal `docker-compose.yml` has to include all required parameters:
-
-```yaml
-version: '2'
-
-services:
-    remark42:
-        image: umputun/remark42:latest
-        restart: always
-        container_name: "remark42"
-        environment:
-            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
-            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
-            - SECRET=abcd-123456-xyz-$%^&           # secret key
-            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
-            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
-        volumes:
-            - ./var:/srv/var                        # persistent volume to store all remark42 data 
-```
-
-#### Register oauth2 providers
-
-Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
-
-##### Google Auth Provider
-
-1.  Create a new project: https://console.developers.google.com/project
-1.  Choose the new project from the top right project dropdown (only if another project is selected)
-1.  In the project Dashboard center pane, choose **"API Manager"**
-1.  In the left Nav pane, choose **"Credentials"**
-1.  In the center pane, choose **"OAuth consent screen"** tab. Fill in **"Product name shown to users"** and hit save.
-1.  In the center pane, choose **"Credentials"** tab.
-    * Open the **"New credentials"** drop down
-    * Choose **"OAuth client ID"**
-    * Choose **"Web application"**
-    * Application name is freeform, choose something appropriate
-    * Authorized origins is your domain ex: `https://remark42.mysite.com`
-    * Authorized redirect URIs is the location of oauth2/callback constructed as domain + `/auth/google/callback`, ex: `https://remark42.mysite.com/auth/google/callback`
-    * Choose **"Create"**
-1.  Take note of the **Client ID** and **Client Secret**
-
-_instructions for google oauth2 setup borrowed from [oauth2_proxy](https://github.com/bitly/oauth2_proxy)_
-
-##### GitHub Auth Provider
-
-1.  Create a new **"OAuth App"**: https://github.com/settings/developers
-1.  Fill **"Application Name"** and **"Homepage URL"** for your site
-1.  Under **"Authorization callback URL"** enter the correct url constructed as domain + `/auth/github/callback`. ie `https://remark42.mysite.com/auth/github/callback`
-1.  Take note of the **Client ID** and **Client Secret**
-
-##### Facebook Auth Provider
-
-1.  From https://developers.facebook.com select **"My Apps"** / **"Add a new App"**
-1.  Set **"Display Name"** and **"Contact email"**
-1.  Choose **"Facebook Login"** and then **"Web"**
-1.  Set "Site URL" to your domain, ex: `https://remark42.mysite.com`
-1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
-1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
-
-##### Yandex Auth Provider
-
-1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
-1.  Fill **"App name"** for your site
-1.  Under **Platforms** select **"Web services"** and enter **"Callback URI #1"** constructed as domain + `/auth/yandex/callback`. ie `https://remark42.mysite.com/auth/yandex/callback`
-1.  Select **Permissions**. You need following permissions only from the **"Yandex.Passport API"** section:
-    * Access to user avatar
-    * Access to username, first name and surname, gender
-1.  Fill out the rest of fields if needed
-1.  Take note of the **ID** and **Password**
-
-For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
-
-#### Initial import from Disqus
-
-1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
-2.  Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
-
-#### Initial import from WordPress
-
-1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 
-2. Move this file to your remark42 host within `./var`
-3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
-
-#### Backup and restore
-
-##### Automatic backups
-Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./var/backup`). Backups kept up to `${MAX_BACKUP_FILES}` (default 10). Each backup file contains exported and gzipped content, i.e., all comments. At any point, the user can restore such backup and revert all comments to the desirable state. Note: restore procedure cleans the current data store and replaces all comments with comments from the backup file.
-
-For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
-
-`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
-
-##### Manual backup
-
-In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
-
-`docker exec -it remark42 backup -s {your site id}`
-
-##### Restore from backup
-
-Restore will clean all comments first and then will processed with complete import from a given file.
-
-`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
-
-##### Backup format
-
-Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
-unmarshaled from `Comment` struct (see below). 
-
-#### Admin users
-
-Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line. 
-
-```
-    environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
-```
-
-To get user id just login and click on your username or any other user you want to promote to admins. 
-It will expand login info and show full user ID.
-
-### Setup on your website
-
-#### Comments
-
-It's a main widget which renders list of comments. 
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID',
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
-    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
-    theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
-    page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add this node in the place where you want to see Remark42 widget:
-
-```html
-<div id="remark42"></div>
-``` 
-
-After that widget will be rendered inside this node.
-
-##### Themes
-
-Right now Remark has two themes: light and dark.
-You can pick one using configuration object,
-but there is also a possibility to switch between themes in runtime.
-For this purpose Remark adds to `window` object named `REMARK42`, 
-which contains function `changeTheme`.
-Just call this function and pass a name of the theme that you want to turn on:
-
-```js
-window.REMARK42.changeTheme('light');
-```      
-
-#### Last comments
-
-It's a widget which renders list of last comments from your site.
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add this node in the place where you want to see last comments widget:
-
-```html
-<div class="remark42__last-comments" data-max="50"></div>
-```
-
-`data-max` sets the max amount of comments (default: `15`).
-
-#### Counter
-
-It's a widget which renders a number of comments for the specified page.
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID',
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add a node like this in the place where you want to see a number of comments:
-
-```html
-<span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
-```
-
-You can use as many nodes like this as you need to. 
-The script will found all them by the class `remark__counter`, 
-and it will use `data-url` attribute to define the page with comments.
-
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
-
-## Build from the source
-
-- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
-- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable 
- `remark42` file with everything embedded.
- 
-## Development
-
-You can use fully functional local version to develop and test both frontend & backend.
-
-To bring it up run:
-
-```bash
-# if you mainly work on backend
-docker-compose -f compose-dev-backend.yml build
-docker-compose -f compose-dev-backend.yml up
-# if you mainly work on frontend
-docker-compose -f compose-dev-frontend.yml build
-docker-compose -f compose-dev-frontend.yml up
-```
-
-It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
-To access UI demo page go to `127.0.0.1:8080/web`.
-By default, you would be logged in as `dev_user` which defined as admin. 
-You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
-
-Backend docker compose config by default skips running frontend related tests. 
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build. 
-
-### Backend development
-
-In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
-
-To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
-It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
-`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
-
-To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
-`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
-
-### Frontend development
-
-#### Build
-
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
-* run `npm install` inside `./web`;
-* run `npm run build` there;
-* result files will be saved in `./web/public`.
-
-**Note** Running `npm install` will set up precommit hooks into your git repository.
-It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
-
-#### Devserver
-
-For local development mode with Hot Reloading use `npm start` instead of `npm run build`.
-In this case `webpack` will serve files using `webpack-dev-server` on `localhost:9000`.
-By visiting `127.0.0.1:9000/web` you will get a page with main comments widget
-communicating with demo server backend running on `https://demo.remark42.com`.
-But you will not be able to login with any oauth providers due to security reasons.
-
-You can attach to locally running backend by providing `REMARK_URL` environment variable.
-```sh
-npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
-```
-
-Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
-[Redux](https://github.com/zalmoxisus/redux-devtools-extension).
-
-## API
-
-### Authorization
-
-* `GET /auth/{provider}/login?from=http://url&site=site_id&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
-* `GET /auth/logout` - logout
-
-```go
-type User struct {
-    Name    string `json:"name"`
-    ID      string `json:"id"`
-    Picture string `json:"picture"`
-    Admin   bool   `json:"admin"`
-    Blocked bool   `json:"block"`
-    Verified bool  `json:"verified"`
-}
-```
-
-_currently supported providers are `google`, `facebook`, `github` and `yandex`_
-
-### Commenting
-
-* `POST /api/v1/comment` - add a comment. _auth required_
-
-```go
-type Comment struct {
-    ID        string          `json:"id"`      // comment ID, read only
-    ParentID  string          `json:"pid"`     // parent ID
-    Text      string          `json:"text"`    // comment text, after md processing
-    Orig      string          `json:"orig"`    // original comment text
-    User      User            `json:"user"`    // user info, read only
-    Locator   Locator         `json:"locator"` // post locator
-    Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
-    Timestamp time.Time       `json:"time"`    // time stamp, read only
-    Pin       bool            `json:"pin"`     // pinned status, read only
-    Delete    bool            `json:"delete"`  // delete status, read only
-    PostTitle string          `json:"title"`   // post title
-}
-
-type Locator struct {
-    SiteID string `json:"site"`     // site id
-    URL    string `json:"url"`      // post url
-}
-```
-
-* `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
-
-* `GET /api/v1/find?site=site-id&url=post-url&sort=fld&format=tree|plain` - find all comments for given post
-
-This is the primary call used by UI to show comments for given post. It can return comments in two formats - `plain` and `tree`.
-In plain format result will be sorted list of `Comment`. In tree format this is going to be tree-like object with this structure:
-
-```go
-type Tree struct {
-    Nodes []Node `json:"comments"`
-    Info  store.PostInfo `json:"info,omitempty"`
-}
-
-type Node struct {
-    Comment store.Comment `json:"comment"`
-    Replies []Node        `json:"replies,omitempty"`
-}
-```
-
-Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
-
-* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
-
-```go
- 	type EditRequest struct {
- 		Text    string `json:"text"`    // updated text
- 		Summary string `json:"summary"` // optional, summary of the edit
- 		Delete  bool   `json:"delete"`  // delete flag 
- 	}{}
-```
-
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
-* `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
-* `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
-  ```go
-  type response struct {
-      Comments []store.Comment  `json:"comments"`
-      Count    int              `json:"count"`
-  }{}
-  ```
-* `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
-* `POST /api/v1/count?site=siteID` - get number of comments for posts from post body (list of post IDs)
-* `GET /api/v1/list?site=site-id&limit=5&skip=2` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
-  ```go
-  type PostInfo struct {
-      URL   string      `json:"url"`
-      Count int         `json:"count"`
-      ReadOnly bool     `json:"read_only,omitempty"`
-      FirstTS time.Time `json:"first_time,omitempty"`
-      LastTS  time.Time `json:"last_time,omitempty"`
-  }
-  ```
-* `GET /api/v1/user` - get user info, _auth required_
-* `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_
-* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream  _auth required_
-* `POST /api/v1/deleteme?site=site-id` - request deletion of user data. _auth required_
-* `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
-
-  ```go
-  type config struct {
-      Version       string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
-      Admins        []string `json:"admins"`
-      Auth          []string `json:"auth_providers"`
-      LowScore      int      `json:"low_score"`
-      CriticalScore int      `json:"critical_score"`
-  }
-  ``` 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
-  
-### RSS feeds
-  
-* `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
-* `GET /api/v1/rss/site?site=site-id` - rss feed for given site
-* `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
-
-### Admin
-
-* `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
-* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1&ttl=7d` - block or unblock user with optional ttl (default=permanent)
-* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids
-  ```go
-  type BlockedUser struct {
-      ID        string    `json:"id"`
-      Name      string    `json:"name"`
-      Until     time.Time `json:"time"`
-  }
-  ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
-* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
-* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
-* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
-* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
-* `PUT /api/v1/admin/readonly?site=site-id&url=post-url&ro=1` - set read-only status
-* `PUT /api/v1/admin/verify/{userid}?site=site-id&verified=1` - set verified status
-* `GET /api/v1/admin/deleteme?token=token` - process deleteme user's request
-
-_all admin calls require auth and admin privilege_
-
-## Privacy 
-
-* Remark42 is trying to be very sensitive to any private or semi-private information.
-* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
-* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
-* There is no tracking of any sort.
-* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF.
-* There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
-* There are no third-party analytic services involved.
-* User can request all information remark42 knows about and export to gz file.
-* Supported complete cleanup of all information related to user's activity.
-* Cookie lifespan can be restricted to session-only. 
-* All potentially sensitive data stored by remark42 hashed and encrypted.
-
-## Technical details
-
-* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `STORE_BOLT_PATH`
-* Each site stored in a separate boltbd file.
-* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_FS_PATH` should be transferred. Optionally, boltdb can be used to store avatars as well.
-* Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
-* Authentication implemented with [go-pkgz/auth](https://github.com/go-pkgz/auth) stored in a cookie. It uses HttpOnly, secure cookies.
-* All heavy REST calls cached internally in LRU cache limited by `CACHE_MAX_ITEMS` and `CACHE_MAX_SIZE` with [go-pkgz/rest](https://github.com/go-pkgz/rest)
-* User's activity throttled globally (up to 1000 simultaneous requests) and limited locally (per user, usually up to 10 req/sec)
-* Request timeout set to 60sec
-* Admin authentication (`--admin-password` set) allows to hit remark42 API without social login and with admin privileges. Adds basic-auth for username: `admin`, password: `${ADMIN_PASSWD}`.
-* User can vote for the comment multiple times but only to change the vote. Double-voting not allowed.
-* User can edit comments in 5 mins (configurable) window after creation.
-* User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
-* All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
-* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
-* Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.
+* [A Helm chart for Remark42 on Kubernetes](https://github.com/groundhog2k/helm-charts/tree/master/charts/remark42)
+* [django-remark42](https://github.com/andrewp-as-is/django-remark42.py)

SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| current | :white_check_mark: |
+| >=1.6.x | :white_check_mark: |
+| <1.5.x  | :x:                |
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to umputun@gmail.com. You will receive a response from us within 48 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.

backend/.golangci.yml

@@ -0,0 +1,69 @@
+run:
+  timeout: 5m
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+# TODO: feel free to remove these excludes and fix the code
+      - hugeParam
+      - rangeValCopy
+
+linters:
+  enable:
+    - bodyclose
+    - megacheck
+    - revive
+    - govet
+    - unconvert
+    - megacheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - unused
+    - typecheck
+    - ineffassign
+    - stylecheck
+    - gochecknoinits
+    - exportloopref
+    - gocritic
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
+  disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "package-comments: should have a package comment"
+      linters:
+        - revive
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false

backend/Gopkg.lock

@@ -1,429 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter",
-  ]
-  pruneopts = "UT"
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
-  name = "github.com/globalsign/mgo"
-  packages = [
-    ".",
-    "bson",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = "UT"
-  revision = "113d3961e7311526535a1ef7042196563d442761"
-  version = "r2018.06.15"
-
-[[projects]]
-  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware",
-  ]
-  pruneopts = "UT"
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
-  name = "github.com/go-chi/cors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dba6525398619dead495962a916728e7ee2ca322"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:d31c72099b9e195785933d11441bc92cce9b2c421f8fca712a7eaf36a8e55a47"
-  name = "github.com/go-pkgz/auth"
-  packages = [
-    ".",
-    "avatar",
-    "logger",
-    "middleware",
-    "provider",
-    "token",
-  ]
-  pruneopts = "UT"
-  revision = "92256b685196d26a346dfbedea12898fe589b0eb"
-  version = "v0.4.0"
-
-[[projects]]
-  digest = "1:1933dabfb0e07548ed9684fdef857cc6f2413bbbb74ba838a7c1f77407b8fd90"
-  name = "github.com/go-pkgz/lgr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8f56fccc1a15dfe9c7f72e049fc87cc378205b97"
-  version = "v0.2.2"
-
-[[projects]]
-  digest = "1:c509e3f646c48148f2239fa66d5cd62785dfd057b5d2b46972e45b60049c19ed"
-  name = "github.com/go-pkgz/mongo"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ee1e9a167094f8cdbc27407907aa694fff350ec3"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
-  name = "github.com/go-pkgz/repeater"
-  packages = [
-    ".",
-    "strategy",
-  ]
-  pruneopts = "UT"
-  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:9aba5c95373481f118e57e9740d9e82f86802d86849809ab89324bcb4f236451"
-  name = "github.com/go-pkgz/rest"
-  packages = [
-    ".",
-    "cache",
-    "logger",
-  ]
-  pruneopts = "UT"
-  revision = "e7d08d0194d613b8854de2e487bf7732500fa153"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = "UT"
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
-  name = "github.com/nullrocks/identicon"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7875f45b0022edded6377e40639d8aa620193a62"
-
-[[projects]]
-  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  pruneopts = "UT"
-  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
-  version = "v0.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
-  name = "golang.org/x/crypto"
-  packages = [
-    "acme",
-    "acme/autocert",
-  ]
-  pruneopts = "UT"
-  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64",
-  ]
-  pruneopts = "UT"
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom",
-  ]
-  pruneopts = "UT"
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex",
-  ]
-  pruneopts = "UT"
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = "UT"
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = "UT"
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/PuerkitoBio/goquery",
-    "github.com/coreos/bbolt",
-    "github.com/dgrijalva/jwt-go",
-    "github.com/didip/tollbooth",
-    "github.com/didip/tollbooth_chi",
-    "github.com/globalsign/mgo",
-    "github.com/globalsign/mgo/bson",
-    "github.com/go-chi/chi",
-    "github.com/go-chi/chi/middleware",
-    "github.com/go-chi/cors",
-    "github.com/go-chi/render",
-    "github.com/go-pkgz/auth",
-    "github.com/go-pkgz/auth/avatar",
-    "github.com/go-pkgz/auth/provider",
-    "github.com/go-pkgz/auth/token",
-    "github.com/go-pkgz/lgr",
-    "github.com/go-pkgz/mongo",
-    "github.com/go-pkgz/repeater",
-    "github.com/go-pkgz/rest",
-    "github.com/go-pkgz/rest/cache",
-    "github.com/go-pkgz/rest/logger",
-    "github.com/google/uuid",
-    "github.com/gorilla/feeds",
-    "github.com/hashicorp/go-multierror",
-    "github.com/jessevdk/go-flags",
-    "github.com/microcosm-cc/bluemonday",
-    "github.com/patrickmn/go-cache",
-    "github.com/pkg/errors",
-    "github.com/rakyll/statik/fs",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/acme/autocert",
-    "golang.org/x/net/html",
-    "gopkg.in/russross/blackfriday.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/Gopkg.toml

@@ -1,44 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-
-[[constraint]]
-  name = "github.com/globalsign/mgo"
-  version = "r2018.06.15"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-
-
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"

backend/README.md

@@ -0,0 +1 @@
+../site/src/docs/contributing/backend/index.md

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,29 @@
+FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+FROM umputun/baseimage:app-v1.9.2
+
+ARG GITHUB_SHA
+
+LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
+      org.opencontainers.image.description="Remark42 comment engine example JRPC memory store" \
+      org.opencontainers.image.documentation="https://github.com/umputun/remark42/tree/master/backend/_example/memory_store" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.title="Remark42 JRPC example memory store" \
+      org.opencontainers.image.url="https://remark42.com/" \
+      org.opencontainers.image.revision="${GITHUB_SHA}"
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,12 @@
+# sample store implementation
+
+`memory_store` illustrates how to make a custom storage plugin for remark42.
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. `docker-compose -f compose-dev-memstore.yml build`
+1. `docker-compose -f compose-dev-memstore.yml up`
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key(_ string) (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, fmt.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", fmt.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, fmt.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return fmt.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated++ // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,573 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	mu        sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", fmt.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", fmt.Errorf("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, fmt.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, fmt.Errorf("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, fmt.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, fmt.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail, engine.UserTelegram:
+		if req.UserID == "" {
+			return nil, fmt.Errorf("userid cannot be empty in request for single detail")
+		}
+
+		m.mu.Lock()
+		defer m.mu.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.mu.Lock()
+			defer m.mu.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, fmt.Errorf("unsupported request with userdetail all")
+	default:
+		return nil, fmt.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return fmt.Errorf("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return fmt.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return fmt.Errorf("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	if err != nil {
+		return false, fmt.Errorf("failed to set flag %+v: %w", req, err)
+	}
+	return status, nil
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		case engine.UserTelegram:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	case engine.UserTelegram:
+		entry.Details.Telegram = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.UserTelegram:
+		entry.Details.Telegram = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, fmt.Errorf("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID != comment.ID || c.Locator != comment.Locator {
+			continue
+		}
+		c.Text = comment.Text
+		c.Orig = comment.Orig
+		c.Score = comment.Score
+		c.Votes = comment.Votes
+		c.Pin = comment.Pin
+		c.Deleted = comment.Deleted
+		c.User = comment.User
+		comments[i] = c
+		m.posts[comment.Locator.SiteID] = comments
+		return nil
+	}
+	return fmt.Errorf("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,819 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds = make([]string, len(blockedList))
+	for i, x := range blockedList {
+		blockedIds[i] = x.ID
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_UserDetailAll(t *testing.T) {
+	b := prepMem(t)
+
+	val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails})
+	require.NoError(t, err)
+	require.Nil(t, val)
+}
+
+func TestMemData_UserDetailErrors(t *testing.T) {
+	b := prepMem(t)
+
+	val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.UserEmail, Update: "value1"})
+	require.EqualError(t, err, "userid cannot be empty in request for single detail")
+	require.Nil(t, val)
+
+	val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails, Update: "value1"})
+	require.EqualError(t, err, "unsupported request with userdetail all")
+	require.Nil(t, val)
+
+	val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: "bad"})
+	require.EqualError(t, err, "unsupported detail \"bad\"")
+	require.Nil(t, val)
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createEmailUser    = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readEmailUser      = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		createTelegramUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram, Update: "value1"}
+		readTelegramUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram}
+		emailSet           = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset         = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+		telegramSet        = []engine.UserDetailEntry{{UserID: "user1", Telegram: "value1"}}
+		telegramUnset      = []engine.UserDetailEntry{{UserID: "user1", Telegram: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readEmailUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: createTelegramUser, expected: telegramSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: readTelegramUser, expected: telegramSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: readTelegramUser, expected: telegramUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readEmailUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	mu            sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.mu.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.mu.Unlock()
+
+	return nil
+}
+
+// ResetCleanupTimer resets cleanup timer for the image
+func (m *MemImage) ResetCleanupTimer(id string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if _, ok := m.insertTime[id]; ok {
+		m.insertTime[id] = time.Now()
+		return nil
+	}
+	return fmt.Errorf("image %s not found", id)
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.mu.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.mu.RUnlock()
+	if !ok {
+		return nil, fmt.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.mu.RLock()
+	img, ok := m.imagesStaging[id]
+	m.mu.RUnlock()
+	if !ok {
+		return fmt.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.mu.Lock()
+	m.images[id] = img
+	m.mu.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.mu.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.mu.RUnlock()
+
+	m.mu.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.mu.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.mu.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.mu.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	svc.ResetCleanupTimer(id)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,63 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: "2"
+
+services:
+  remark42:
+    build:
+      context: ../../..
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42-dev"
+    hostname: "remark42-dev"
+    restart: always
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: ../../..
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,42 @@
+module github.com/umputun/remark42/memory_store
+
+go 1.17
+
+require (
+	github.com/go-pkgz/jrpc v0.3.0
+	github.com/go-pkgz/lgr v0.10.4
+	github.com/jessevdk/go-flags v1.5.0
+	github.com/stretchr/testify v1.8.0
+	github.com/umputun/remark42/backend v1.10.1
+)
+
+require (
+	github.com/Depado/bfchroma/v2 v2.0.0 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/ajg/form v1.5.1 // indirect
+	github.com/alecthomas/chroma/v2 v2.3.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/didip/tollbooth/v7 v7.0.1 // indirect
+	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/go-chi/chi/v5 v5.0.7 // indirect
+	github.com/go-chi/render v1.0.2 // indirect
+	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
+	github.com/go-pkgz/rest v1.16.0 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.20 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/image v0.0.0-20220617043117-41969df76e82 // indirect
+	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/umputun/remark42/backend => ../../

backend/_example/memory_store/go.sum

@@ -0,0 +1,90 @@
+github.com/Depado/bfchroma/v2 v2.0.0 h1:IRpN9BPkNwEpR6w1ectIcNWOuhDSLx+8f1pn83fzxx8=
+github.com/Depado/bfchroma/v2 v2.0.0/go.mod h1:wFwW/Pw8Tnd0irzgO9Zxtxgzp3aPS8qBWlyadxujxmw=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/chroma/v2 v2.3.0 h1:83xfxrnjv8eK+Cf8qZDzNo3PPF9IbTWHs7z28GY6D0U=
+github.com/alecthomas/chroma/v2 v2.3.0/go.mod h1:mZxeWZlxP2Dy+/8cBob2PYd8O2DwNAzave5AY7A2eQw=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/didip/tollbooth/v7 v7.0.0/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
+github.com/didip/tollbooth/v7 v7.0.1 h1:TkT4sBKoQoHQFPf7blQ54iHrZiTDnr8TceU+MulVAog=
+github.com/didip/tollbooth/v7 v7.0.1/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
+github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f h1:jtKwihcLmUC9BAhoJ9adCUqdSSZcOdH2KL7mPTUm2aw=
+github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f/go.mod h1:q9C80dnsuVRP2dAskjnXRNWdUJqtGgwG9wNrzt0019s=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
+github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg=
+github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
+github.com/go-pkgz/expirable-cache v0.1.0 h1:3bw0m8vlTK8qlwz5KXuygNBTkiKRTPrAGXU0Ej2AC1g=
+github.com/go-pkgz/expirable-cache v0.1.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
+github.com/go-pkgz/jrpc v0.3.0 h1:Fls38KqPsHzvp0FWfivr6cGnncC+iFBodHBqvUPY+0U=
+github.com/go-pkgz/jrpc v0.3.0/go.mod h1:MFtKs75JESiSqVicsQkgN2iDFFuCd3gVT1/vKiwRi00=
+github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
+github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
+github.com/go-pkgz/rest v1.15.6/go.mod h1:KUWAqbDteYGS/CiXftomQsKjtEOifXsJ36Ka0skYbmk=
+github.com/go-pkgz/rest v1.16.0 h1:3An51XJXUq1XtICHvfAEp048ZCbUHBEga3g0WEu3zK0=
+github.com/go-pkgz/rest v1.16.0/go.mod h1:HHlLOt02NJc2sgffXBF6hYVMcRo4Gz3vjg43zTzN7VM=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
+github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/main.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/jessevdk/go-flags"
+
+	"github.com/umputun/remark42/memory_store/accessor"
+	"github.com/umputun/remark42/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
+
+	rpcServer := jrpc.NewServer(
+		opts.API,
+		jrpc.Auth(opts.AuthUser, opts.AuthPasswd),
+		jrpc.WithSignature("remark42-memory", "umputun", revision),
+		jrpc.WithLogger(log.Default()),
+	)
+
+	srv := server.NewRPC(dataStore, adminStore, imgStore, rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID:  "remark",
+		Enabled: true,
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	key, err := s.adm.Key(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,343 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	// verify user
+	verifyFlagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(verifyFlagReq)
+	require.NoError(t, err)
+	assert.Empty(t, flags)
+
+	verifyFlagReq.Update = engine.FlagTrue
+	status, err := re.Flag(verifyFlagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(verifyFlagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+	verifiedUsers := make([]string, 0, len(flags))
+	for _, v := range flags {
+		verifiedUsers = append(verifiedUsers, v.(string))
+	}
+	assert.Equal(t, []string{"u1"}, verifiedUsers)
+
+	// block user
+	blockFlagReq := engine.FlagRequest{
+		Flag:   engine.Blocked,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		TTL: time.Hour,
+	}
+	flags, err = re.ListFlags(blockFlagReq)
+	require.NoError(t, err)
+	assert.Empty(t, flags)
+
+	blockFlagReq.Update = engine.FlagTrue
+	status, err = re.Flag(blockFlagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(blockFlagReq)
+	require.NoError(t, err)
+	assert.NotEmpty(t, flags)
+	blockedUsers := make([]store.BlockedUser, 0, len(flags))
+	for _, v := range flags {
+		blockedUsers = append(blockedUsers, v.(store.BlockedUser))
+	}
+	require.Equal(t, 1, len(blockedUsers))
+	blockedUserInfo := blockedUsers[0]
+	assert.Equal(t, "u1", blockedUserInfo.ID)
+	assert.True(t, blockedUserInfo.Until.After(time.Now().Add(time.Minute*59)), "blocked duration is more than 59m away")
+	assert.True(t, blockedUserInfo.Until.Before(time.Now().Add(time.Minute*61)), "blocked duration is less than 61m away")
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.ResetCleanupTimer(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := io.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// wait for image to expire
+	time.Sleep(time.Millisecond * 50)
+	// reset the time to cleanup
+	err = ri.ResetCleanupTimer(id)
+	assert.NoError(t, err)
+
+	// cleanup, should not affect the new image
+	err = ri.Cleanup(context.TODO(), time.Millisecond*45)
+	assert.NoError(t, err)
+
+	// load after cleanup should succeed
+	_, err = ri.Load(id)
+	assert.NoError(t, err, "image is still on staging because it's cleanup timer was reset")
+
+	// cleanup with short TTL, should remove the image from staging
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/server/rpc.go

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
+// Note: this file can be used as-is in any custom jrpc plugin
+type RPC struct {
+	*jrpc.Server
+	eng engine.Interface
+	adm admin.Store
+	img image.Store
+}
+
+// NewRPC makes RPC instance and register handlers
+func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, img: i, Server: r}
+	res.addHandlers()
+	return res
+}
+
+func (s *RPC) addHandlers() {
+	// data store handlers
+	s.Group("store", jrpc.HandlersGroup{
+		"create":      s.createHndl,
+		"find":        s.findHndl,
+		"get":         s.getHndl,
+		"update":      s.updateHndl,
+		"count":       s.countHndl,
+		"info":        s.infoHndl,
+		"flag":        s.flagHndl,
+		"list_flags":  s.listFlagsHndl,
+		"user_detail": s.userDetailHndl,
+		"delete":      s.deleteHndl,
+		"close":       s.closeHndl,
+	})
+
+	// admin store handlers
+	s.Group("admin", jrpc.HandlersGroup{
+		"key":     s.admKeyHndl,
+		"admins":  s.admAdminsHndl,
+		"email":   s.admEmailHndl,
+		"enabled": s.admEnabledHndl,
+		"event":   s.admEventHndl,
+	})
+
+	// image store handlers
+	s.Group("image", jrpc.HandlersGroup{
+		"save_with_id":        s.imgSaveWithIDHndl,
+		"reset_cleanup_timer": s.imgResetClnTimerHndl,
+		"load":                s.imgLoadHndl,
+		"commit":              s.imgCommitHndl,
+		"cleanup":             s.imgCleanupHndl,
+		"info":                s.imgInfoHndl,
+	})
+}

backend/_example/memory_store/server/rpc_test.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/memory_store/accessor"
+)
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func prepTestStore(t *testing.T) (port int, teardown func()) {
+	mg := accessor.NewMemData()
+	adm := accessor.NewMemAdminStore("secret")
+	img := accessor.NewMemImageStore()
+	s := NewRPC(mg, adm, img, jrpc.NewServer("/test"))
+
+	admRec := accessor.AdminRec{
+		SiteID:  "test-site",
+		IDs:     []string{"id1", "id2"},
+		Email:   "admin@example.com",
+		Enabled: true,
+	}
+	adm.Set("test-site", admRec)
+
+	admRecDisabled := admRec
+	admRecDisabled.Enabled = false
+	adm.Set("test-site-disabled", admRecDisabled)
+
+	port = chooseRandomUnusedPort()
+	go func() {
+		_ = s.Run(port)
+	}()
+
+	waitForHTTPServerStart(port)
+
+	return port, func() {
+		require.NoError(t, s.Shutdown())
+	}
+}

backend/app/cmd/avatar.go

@@ -1,15 +1,13 @@
 package cmd
 
 import (
+	"fmt"
 	"path"
-	"time"
 
-	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -31,22 +28,23 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }
 
 // Execute runs  with AvatarCommand parameters, entry point for "avatar" command
-func (ac *AvatarCommand) Execute(args []string) error {
+func (ac *AvatarCommand) Execute(_ []string) error {
 	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
 
 	src, err := ac.makeAvatarStore(ac.AvatarSrc)
 	if err != nil {
-		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
+		return fmt.Errorf("can't make avatart store for %s: %w", ac.AvatarSrc.Type, err)
 	}
 
 	dst, err := ac.makeAvatarStore(ac.AvatarDst)
 	if err != nil {
-		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
+		return fmt.Errorf("can't make avatart store for %s: %w", ac.AvatarDst.Type, err)
 	}
 
 	if ac.migrator == nil {
@@ -74,28 +72,14 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	switch gr.Type {
 	case "fs":
 		if err := makeDirs(gr.FS.Path); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to create avatar store: %w", err)
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to create avatar store: %w", err)
 		}
 		return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
 	}
-	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
-}
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
+	return nil, fmt.Errorf("unsupported avatar store type %s", gr.Type)
 }

backend/app/cmd/avatar_test.go

@@ -1,54 +1,38 @@
 package cmd
 
 import (
-	"errors"
+	"fmt"
 	"os"
 	"testing"
 
 	"github.com/go-pkgz/auth/avatar"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestAvatar_Execute(t *testing.T) {
-
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
+	defer os.Remove("/tmp/ava-test.db")
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
 	// failed
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
+	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: fmt.Errorf("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test2.db"})
+	require.NoError(t, err)
+	defer os.Remove("/tmp/ava-test2.db")
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }
@@ -59,7 +43,7 @@ type avatarMigratorMock struct {
 	retCount int
 }
 
-func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
+func (a *avatarMigratorMock) Migrate(_, _ avatar.Store) (int, error) {
 	a.called++
 	return a.retCount, a.retError
 }

backend/app/cmd/backup.go

@@ -9,22 +9,20 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 )
 
 // BackupCommand set of flags and command for export
 // ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
 type BackupCommand struct {
-	ExportPath  string        `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
-	ExportFile  string        `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
+	ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
+
+	SupportCmdOpts
 	CommonOpts
 }
 
 // Execute runs export with ExportCommand parameters, entry point for "export" command
-func (ec *BackupCommand) Execute(args []string) error {
+func (ec *BackupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 
@@ -38,19 +36,20 @@ func (ec *BackupCommand) Execute(args []string) error {
 
 	// prepare http client and request
 	client := http.Client{}
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
 	defer cancel()
 	exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s", ec.RemarkURL, ec.Site)
-	req, err := http.NewRequest(http.MethodGet, exportURL, nil)
+	req, err := http.NewRequest(http.MethodGet, exportURL, http.NoBody)
 	if err != nil {
-		return errors.Wrapf(err, "can't make export request for %s", exportURL)
+		return fmt.Errorf("can't make export request for %s: %w", exportURL, err)
 	}
 	req.SetBasicAuth("admin", ec.AdminPasswd)
 
 	// get with timeout
 	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
-		return errors.Wrapf(err, "request failed for %s", exportURL)
+		return fmt.Errorf("request failed for %s: %w", exportURL, err)
 	}
 	defer func() {
 		if err = resp.Body.Close(); err != nil {
@@ -62,18 +61,18 @@ func (ec *BackupCommand) Execute(args []string) error {
 		return responseError(resp)
 	}
 
-	fh, err := os.Create(fname)
+	fh, err := os.Create(fname) //nolint:gosec // harmless
 	if err != nil {
-		return errors.Wrapf(err, "can't create backup file %s", fname)
+		return fmt.Errorf("can't create backup file %s: %w", fname, err)
 	}
-	defer func() {
+	defer func() { //nolint:gosec // false positive on defer without error check when it's checked here
 		if err = fh.Close(); err != nil {
 			log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
 		}
 	}()
 
 	if _, err = io.Copy(fh, resp.Body); err != nil {
-		return errors.Wrapf(err, "failed to write backup file %s", fname)
+		return fmt.Errorf("failed to write backup file %s: %w", fname, err)
 	}
 
 	log.Printf("[INFO] export completed, file %s", fname)

backend/app/cmd/backup_test.go

@@ -2,13 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,13 +24,13 @@ func TestBackup_Execute(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	defer os.Remove("/tmp/remark-test.export")
 
-	data, err := ioutil.ReadFile("/tmp/remark-test.export")
-	require.Nil(t, err)
+	data, err := os.ReadFile("/tmp/remark-test.export")
+	require.NoError(t, err)
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
@@ -49,7 +48,7 @@ func TestBackup_ExecuteFailedStatus(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
 }
@@ -68,7 +67,7 @@ func TestBackup_ExecuteFailedWrite(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
 		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
 }

backend/app/cmd/cleanup.go

@@ -9,21 +9,20 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // CleanupCommand set of flags and command for cleanup
 type CleanupCommand struct {
-	Site        string   `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Dry         bool     `long:"dry" description:"dry mode, will not remove comments"`
-	From        string   `long:"from" description:"from yyyymmdd"`
-	To          string   `long:"to" description:"from yyyymmdd"`
-	BadWords    []string `short:"w" long:"bword" description:"bad word(s)"`
-	BadUsers    []string `short:"u" long:"buser" description:"bad user(s)"`
-	AdminPasswd string   `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
-	SetTitle    bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+	Dry      bool     `long:"dry" description:"dry mode, will not remove comments"`
+	From     string   `long:"from" description:"from yyyymmdd"`
+	To       string   `long:"to" description:"from yyyymmdd"`
+	BadWords []string `short:"w" long:"bword" description:"bad word(s)"`
+	BadUsers []string `short:"u" long:"buser" description:"bad user(s)"`
+	SetTitle bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+
+	SupportCmdOpts
 	CommonOpts
 }
 
@@ -34,12 +33,12 @@ var (
 
 // Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
 // This command uses provided flags to detect and remove junk comments
-func (cc *CleanupCommand) Execute(args []string) error {
+func (cc *CleanupCommand) Execute(_ []string) error {
 	log.Printf("[INFO] cleanup for site %s", cc.Site)
 
 	posts, err := cc.postsInRange(cc.From, cc.To)
 	if err != nil {
-		return errors.Wrap(err, "can't get posts")
+		return fmt.Errorf("can't get posts: %w", err)
 	}
 	log.Printf("[DEBUG] got %d posts", len(posts))
 
@@ -55,7 +54,6 @@ func (cc *CleanupCommand) Execute(args []string) error {
 			cc.procTitles(comments)
 		} else {
 			spamComments += cc.procSpam(comments)
-
 		}
 	}
 
@@ -100,7 +98,7 @@ func (cc *CleanupCommand) procTitles(comments []store.Comment) {
 func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, error) {
 	posts, err := cc.listPosts()
 	if err != nil {
-		return nil, errors.Wrapf(err, "can't list posts for %s", cc.Site)
+		return nil, fmt.Errorf("can't list posts for %s: %w", cc.Site, err)
 	}
 
 	from, to := defaultFrom, defaultTo
@@ -108,14 +106,14 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 	if fromS != "" {
 		from, err = time.ParseInLocation("20060102", fromS, time.Local)
 		if err != nil {
-			return nil, errors.Wrap(err, "can't parse --from")
+			return nil, fmt.Errorf("can't parse --from: %w", err)
 		}
 	}
 
 	if toS != "" {
 		to, err = time.ParseInLocation("20060102", toS, time.Local)
 		if err != nil {
-			return nil, errors.Wrap(err, "can't parse --to")
+			return nil, fmt.Errorf("can't parse --to: %w", err)
 		}
 	}
 
@@ -131,36 +129,39 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL)
+	client := http.Client{Timeout: 30 * time.Second}
+	defer client.CloseIdleConnections()
+	r, err := client.Get(listURL)
 	if err != nil {
-		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
+		return nil, fmt.Errorf("get request failed for list of posts, site %s: %w", cc.Site, err)
 	}
 	defer func() { _ = r.Body.Close() }()
 
 	if r.StatusCode != 200 {
-		return nil, errors.Errorf("request %s failed with status %d", listURL, r.StatusCode)
+		return nil, fmt.Errorf("request %s failed with status %d", listURL, r.StatusCode)
 	}
 
 	list := []store.PostInfo{}
 	if err = json.NewDecoder(r.Body).Decode(&list); err != nil {
-		return nil, errors.Wrapf(err, "can't decode list of posts for site %s", cc.Site)
+		return nil, fmt.Errorf("can't decode list of posts for site %s: %w", cc.Site, err)
 	}
 	return list, nil
 }
 
 // get all comments for post url via /find?site=siteID&url=post-url&format=[tree|plain]
 func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error) {
-
 	commentsURL := fmt.Sprintf("%s/api/v1/find?site=%s&url=%s&format=plain", cc.RemarkURL, cc.Site, postURL)
 
 	var r *http.Response
 	var err error
 
 	// handle 429 error from limiter
+	client := http.Client{Timeout: 30 * time.Second}
+	defer client.CloseIdleConnections()
 	for {
-		r, err = http.Get(commentsURL)
+		r, err = client.Get(commentsURL)
 		if err != nil {
-			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
+			return nil, fmt.Errorf("get request failed for comments, %s: %w", postURL, err)
 		}
 		if r.StatusCode == http.StatusTooManyRequests {
 			_ = r.Body.Close()
@@ -173,7 +174,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 	defer func() { _ = r.Body.Close() }()
 
 	if r.StatusCode != http.StatusOK {
-		return nil, errors.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
+		return nil, fmt.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
 	}
 
 	commentsWithInfo := struct {
@@ -181,59 +182,58 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
-		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+		return nil, fmt.Errorf("can't decode list of comments for %s: %w", postURL, err)
 	}
 	return commentsWithInfo.Comments, nil
 }
 
 // deleteComment with DELETE /admin/comment/{id}?site=siteID&url=post-url
-func (cc *CleanupCommand) deleteComment(c store.Comment) error {
-
+func (cc *CleanupCommand) deleteComment(c store.Comment) error { //nolint:dupl // not worth combining
 	deleteURL := fmt.Sprintf("%s/api/v1/admin/comment/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
-	req, err := http.NewRequest("DELETE", deleteURL, nil)
+	req, err := http.NewRequest("DELETE", deleteURL, http.NoBody)
 	if err != nil {
-		return errors.Wrapf(err, "failed to make delete request for comment %s, %s", c.ID, c.Locator.URL)
+		return fmt.Errorf("failed to make delete request for comment %s, %s: %w", c.ID, c.Locator.URL, err)
 	}
 	req.SetBasicAuth("admin", cc.AdminPasswd)
 
 	client := http.Client{}
+	defer client.CloseIdleConnections()
 	r, err := client.Do(req)
 	if err != nil {
-		return errors.Wrapf(err, "delete request failed for comment %s, %s", c.ID, c.Locator.URL)
+		return fmt.Errorf("delete request failed for comment %s, %s: %w", c.ID, c.Locator.URL, err)
 	}
 	defer func() { _ = r.Body.Close() }()
 	if r.StatusCode != http.StatusOK {
-		return errors.Errorf("delete request failed with status %s", r.Status)
+		return fmt.Errorf("delete request failed with status %s", r.Status)
 	}
 	return nil
 }
 
 // setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
-func (cc *CleanupCommand) setTitle(c store.Comment) error {
-
+func (cc *CleanupCommand) setTitle(c store.Comment) error { //nolint:dupl // not worth combining
 	titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
-	req, err := http.NewRequest("PUT", titleURL, nil)
+	req, err := http.NewRequest("PUT", titleURL, http.NoBody)
 	if err != nil {
-		return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
+		return fmt.Errorf("failed to make title request for comment %s, %s: %w", c.ID, c.Locator.URL, err)
 	}
 	req.SetBasicAuth("admin", cc.AdminPasswd)
 
 	client := http.Client{}
+	defer client.CloseIdleConnections()
 	r, err := client.Do(req)
 	if err != nil {
-		return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
+		return fmt.Errorf("title request failed for comment %s, %s: %w", c.ID, c.Locator.URL, err)
 	}
 	defer func() { _ = r.Body.Close() }()
 	if r.StatusCode != http.StatusOK {
-		return errors.Errorf("title request failed with status %s", r.Status)
+		return fmt.Errorf("title request failed with status %s", r.Status)
 	}
 	return nil
 }
 
 // isSpam calculates spam's probability as a score
-func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
-
+func (cc *CleanupCommand) isSpam(comment store.Comment) (isSpam bool, spamScore float64) {
 	badWord := func(txt string) float64 {
 		res := 0.0
 		for _, w := range cc.BadWords {

backend/app/cmd/cleanup_test.go

@@ -9,12 +9,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/go-chi/chi"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/go-chi/chi/v5"
+	"github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 type cleanedComments struct {
@@ -46,6 +46,7 @@ func TestCleanup_IsSpam(t *testing.T) {
 	}
 
 	for n, tt := range tbl {
+		tt := tt
 		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
 		t.Run(checkName, func(t *testing.T) {
 			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
@@ -58,7 +59,6 @@ func TestCleanup_IsSpam(t *testing.T) {
 }
 
 func TestCleanup_postsInRange(t *testing.T) {
-
 	r := chi.NewRouter()
 	cleanupRoutes(t, r, nil)
 	ts := httptest.NewServer(r)
@@ -68,7 +68,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	posts, err := cmd.postsInRange("20181218", "20181219")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts))
@@ -78,7 +78,7 @@ func TestCleanup_postsInRange(t *testing.T) {
 	assert.Equal(t, 3, len(posts))
 
 	_, err = cmd.postsInRange("xxx", "yyy")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestCleanup_listComments(t *testing.T) {
@@ -91,7 +91,7 @@ func TestCleanup_listComments(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	comments, err := cmd.listComments("http://test.com/post1")
 	assert.NoError(t, err)
@@ -118,7 +118,7 @@ func TestCleanup_ExecuteSpam(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
 		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("deleted %+v", cleaned.ids)
@@ -136,7 +136,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 	t.Logf("set titles for %+v", titledComments.ids)
@@ -144,7 +144,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
 }
 
 func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
-	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
 		list := []store.PostInfo{
@@ -165,9 +165,9 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 			},
 		}
 		require.NoError(t, json.NewEncoder(w).Encode(list))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/find", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "GET", r.Method)
 		require.Equal(t, "remark", r.URL.Query().Get("site"))
 		require.Equal(t, "plain", r.URL.Query().Get("format"))
@@ -193,22 +193,21 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		}
 
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
+	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()
 		c.ids = append(c.ids, r.URL.Path)
 		c.lock.Unlock()
-	}))
-
+	})
 }

backend/app/cmd/cmd.go

@@ -4,7 +4,8 @@ package cmd
 
 import (
 	"bytes"
-	"io/ioutil"
+	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -13,7 +14,6 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 )
 
 // CommonOptionsCommander extends flags.Commander with SetCommon
@@ -21,6 +21,7 @@ import (
 type CommonOptionsCommander interface {
 	SetCommon(commonOpts CommonOpts)
 	Execute(args []string) error
+	HandleDeprecatedFlags() []DeprecatedFlag
 }
 
 // CommonOpts sets externally from main, shared across all commands
@@ -30,14 +31,33 @@ type CommonOpts struct {
 	Revision     string
 }
 
+// SupportCmdOpts is set of commands shared among similar commands like backup/restore and such.
+// Order of fields defines the help command output order.
+type SupportCmdOpts struct {
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	Timeout     time.Duration `long:"timeout" default:"60m" description:"timeout for the command run"`
+}
+
+// DeprecatedFlag contains information about deprecated option
+type DeprecatedFlag struct {
+	Old       string
+	New       string
+	Version   string
+	Collision bool
+}
+
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
-	c.RemarkURL = commonOpts.RemarkURL
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
 	c.SharedSecret = commonOpts.SharedSecret
 	c.Revision = commonOpts.Revision
 }
 
+// HandleDeprecatedFlags sets new flags from deprecated and returns their list
+func (c *CommonOpts) HandleDeprecatedFlags() []DeprecatedFlag { return nil }
+
 // fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
 type fileParser struct {
 	site string
@@ -47,7 +67,6 @@ type fileParser struct {
 
 // parse apply template and also concat path and file. In case if file contains path separator path will be ignored
 func (p *fileParser) parse(now time.Time) (string, error) {
-
 	// file/location parameters my have template masks
 	fileTemplate := struct {
 		YYYYMMDD string
@@ -76,7 +95,7 @@ func (p *fileParser) parse(now time.Time) (string, error) {
 	}
 
 	if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
-		return "", errors.Wrapf(err, "failed to parse %q", fname)
+		return "", fmt.Errorf("failed to parse %q: %w", fname, err)
 	}
 	return bb.String(), nil
 }
@@ -92,18 +111,18 @@ func resetEnv(envs ...string) {
 
 // responseError returns error with status and response body
 func responseError(resp *http.Response) error {
-	body, e := ioutil.ReadAll(resp.Body)
+	body, e := io.ReadAll(resp.Body)
 	if e != nil {
 		body = []byte("")
 	}
-	return errors.Errorf("error response %q, %s", resp.Status, body)
+	return fmt.Errorf("error response %q, %s", resp.Status, body)
 }
 
 // mkdir -p for all dirs
 func makeDirs(dirs ...string) error {
 	for _, dir := range dirs {
-		if err := os.MkdirAll(dir, 0700); err != nil { // If path is already a directory, MkdirAll does nothing
-			return errors.Wrapf(err, "can't make directory %s", dir)
+		if err := os.MkdirAll(dir, 0o700); err != nil { // If path is already a directory, MkdirAll does nothing
+			return fmt.Errorf("can't make directory %s: %w", dir, err)
 		}
 	}
 	return nil

backend/app/cmd/cmd_test.go

@@ -31,7 +31,7 @@ func TestExport_ParseFileName(t *testing.T) {
 	for i, tt := range tbl {
 		r, err := tt.p.parse(now)
 		if tt.err {
-			assert.NotNil(t, err)
+			assert.Error(t, err)
 			continue
 		}
 		assert.Equal(t, tt.res, r, "check #%d", i)

backend/app/cmd/import.go

@@ -5,49 +5,46 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
-	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 )
 
 // ImportCommand set of flags and command for import
 type ImportCommand struct {
-	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
+	Provider  string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint
+
+	SupportCmdOpts
 	CommonOpts
 }
 
 // Execute runs import with ImportCommand parameters, entry point for "import" command
-func (ic *ImportCommand) Execute(args []string) error {
+func (ic *ImportCommand) Execute(_ []string) error {
 	log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
 	resetEnv("SECRET", "ADMIN_PASSWD")
 
 	reader, err := ic.reader(ic.InputFile)
 	if err != nil {
-		return errors.Wrapf(err, "can't open import file %s", ic.InputFile)
+		return fmt.Errorf("can't open import file %s: %w", ic.InputFile, err)
 	}
 
 	client := http.Client{}
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
 	defer cancel()
 	importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s", ic.RemarkURL, ic.Site, ic.Provider)
 	req, err := http.NewRequest(http.MethodPost, importURL, reader)
 	if err != nil {
-		return errors.Wrapf(err, "can't make import request for %s", importURL)
+		return fmt.Errorf("can't make import request for %s: %w", importURL, err)
 	}
 	req.SetBasicAuth("admin", ic.AdminPasswd)
 
 	resp, err := client.Do(req.WithContext(ctx)) // closes request's reader
 	if err != nil {
-		return errors.Wrapf(err, "request failed for %s", importURL)
+		return fmt.Errorf("request failed for %s: %w", importURL, err)
 	}
 	defer func() {
 		if err = resp.Body.Close(); err != nil {
@@ -58,9 +55,9 @@ func (ic *ImportCommand) Execute(args []string) error {
 		return responseError(resp)
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return errors.Wrap(err, "can't get response from importer")
+		return fmt.Errorf("can't get response from importer: %w", err)
 	}
 
 	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
@@ -69,15 +66,15 @@ func (ic *ImportCommand) Execute(args []string) error {
 
 // reader returns reader for file. For .gz file wraps with gunzip
 func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
-	inpFile, err := os.Open(inp)
+	inpFile, err := os.Open(inp) // nolint
 	if err != nil {
-		return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
+		return nil, fmt.Errorf("import failed, can't open %s: %w", inp, err)
 	}
 
 	reader = inpFile
 	if strings.HasSuffix(ic.InputFile, ".gz") {
 		if reader, err = gzip.NewReader(inpFile); err != nil {
-			return nil, errors.Wrap(err, "can't make gz reader")
+			return nil, fmt.Errorf("can't make gz reader: %w", err)
 		}
 	}
 	return reader, nil

backend/app/cmd/import_test.go

@@ -2,26 +2,24 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestImport_Execute(t *testing.T) {
-
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
-		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -34,7 +32,7 @@ func TestImport_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -43,13 +41,12 @@ func TestImport_Execute(t *testing.T) {
 
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
 
 func TestImport_ExecuteFailed(t *testing.T) {
-
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
@@ -61,21 +58,21 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on no such file")
-	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
+	assert.Error(t, err, "fail on no such file")
+	assert.Contains(t, err.Error(), "no such file or directory")
 
 	cmd = ImportCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err, "fail on connection refused")
-	assert.True(t, strings.Contains(err.Error(), "connection refused"))
+	assert.Error(t, err, "fail on connection refused")
+	assert.Contains(t, err.Error(), "connection refused")
 
 	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		log.Printf("%+v", r)
@@ -87,23 +84,22 @@ func TestImport_ExecuteFailed(t *testing.T) {
 	cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
 	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	t.Log(err)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestImport_ExecuteTimeout(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
-		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 		time.Sleep(500 * time.Millisecond)
 		fmt.Fprintln(w, "some response")
 		fmt.Fprintln(w, string(body))
-
 	}))
 	defer ts.Close()
 
@@ -112,8 +108,8 @@ func TestImport_ExecuteTimeout(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "deadline exceeded")
 }

backend/app/cmd/remap.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// RemapCommand set of flags and command for change linkage between comments to
+// different urls based on given rules (input file)
+type RemapCommand struct {
+	InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
+
+	SupportCmdOpts
+	CommonOpts
+}
+
+// Execute runs (re)mapper with RemapCommand parameters, entry point for "remap" command
+func (rc *RemapCommand) Execute(_ []string) error {
+	log.Printf("[INFO] start remap, site %s, file with rules %s", rc.Site, rc.InputFile)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	rulesReader, err := os.Open(rc.InputFile)
+	if err != nil {
+		return fmt.Errorf("cant open file %s: %w", rc.InputFile, err)
+	}
+
+	client := http.Client{}
+	defer client.CloseIdleConnections()
+	ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
+	defer cancel()
+	remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
+	req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
+	if err != nil {
+		return fmt.Errorf("can't make remap request for %s: %w", remapURL, err)
+	}
+	req.SetBasicAuth("admin", rc.AdminPasswd)
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return fmt.Errorf("request failed for %s: %w", remapURL, err)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("can't get response: %w", err)
+	}
+
+	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+	return nil
+}

backend/app/cmd/remap_test.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemap_Execute(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(202)
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}

backend/app/cmd/restore.go

@@ -11,9 +11,7 @@ type RestoreCommand struct {
 	ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
 	ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
 
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	SupportCmdOpts
 	CommonOpts
 }
 
@@ -29,12 +27,10 @@ func (rc *RestoreCommand) Execute(args []string) error {
 		return err
 	}
 	importer := ImportCommand{
-		InputFile:   fname,
-		Site:        rc.Site,
-		Provider:    "native",
-		Timeout:     rc.Timeout,
-		AdminPasswd: rc.AdminPasswd,
-		CommonOpts:  rc.CommonOpts,
+		InputFile:      fname,
+		Provider:       "native",
+		SupportCmdOpts: rc.SupportCmdOpts,
+		CommonOpts:     rc.CommonOpts,
 	}
 	return importer.Execute(args)
 }

backend/app/cmd/restore_test.go

@@ -2,24 +2,23 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestRestore_Execute(t *testing.T) {
-
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "native", r.URL.Query().Get("provider"))
-		body, err := ioutil.ReadAll(r.Body)
-		assert.Nil(t, err)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
 
 		fmt.Fprintln(w, "some response")
@@ -32,7 +31,7 @@ func TestRestore_Execute(t *testing.T) {
 
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt", "--admin-passwd=secret"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }

backend/app/cmd/server_test.go

@@ -4,163 +4,249 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
-	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/golang-jwt/jwt"
+	"github.com/jessevdk/go-flags"
+	"go.uber.org/goleak"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	defer http.DefaultClient.CloseIdleConnections()
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 10 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	body, _ = ioutil.ReadAll(resp.Body)
+	body, _ = io.ReadAll(resp.Body)
 	t.Log(string(body))
 
-	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+	email, err := app.dataService.AdminStore.Email("")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
+	cancel()
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
-	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
-	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
+	providers := app.restSrv.Authenticator.Providers()
+	require.Equal(t, 9+1, len(providers), "extra auth provider")
+	assert.Equal(t, "dev", providers[len(providers)-2].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	defer http.DefaultClient.CloseIdleConnections()
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
 	app.Wait()
 }
 
-func TestServerApp_WithMongo(t *testing.T) {
+func TestServerApp_AnonMode(t *testing.T) {
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
+		o.Auth.Anonymous = true
+		return o
+	})
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
-
-	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
-	// prepare options
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
-		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
-	require.Nil(t, err)
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation = "/tmp"
-
-	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
-
-	defer func() {
-		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, err)
-		conn := mongo.NewConnection(s, "test_remark", "")
-		_ = conn.WithDB(func(dbase *mgo.Database) error {
-			assert.NoError(t, dbase.DropDatabase())
-			return nil
-		})
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
+
+	providers := app.restSrv.Authenticator.Providers()
+	require.Equal(t, 9+1, len(providers), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", providers[len(providers)-1].Name(), "anon auth provider")
+
+	client := http.Client{Timeout: 10 * time.Second}
+	defer client.CloseIdleConnections()
 
 	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	// try to login with good name
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as good anonymous
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims := getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	assert.False(t, claims.User.BoolAttr("blocked"), "should not be blocked")
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+
+	// try to login with non-latin name
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to login with bad name
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with short name
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with name what have space in prefix
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with name what have space in suffix
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with long name
+	time.Sleep(time.Second)
+	ln := strings.Repeat("x", 65)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
+	// try to login with admin name
+	time.Sleep(time.Second)
+	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// try to add a comment as anonymous with admin name
+	time.Sleep(time.Second)
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
+
+	tkn, claims = getAuthFromCookie(t, app, resp)
+	require.NotEmpty(t, tkn)
+	assert.True(t, claims.User.BoolAttr("blocked"), "should be blocked")
+	req.Header.Add("X-JWT", tkn)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+
+	cancel()
 	app.Wait()
 }
 
+func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (tkn string, claims token.Claims) {
+	var err error
+	for _, c := range resp.Cookies() {
+		if c.Name == "JWT" {
+			tkn = c.Value
+			claims, err = app.restSrv.Authenticator.TokenService().Parse(c.Value)
+			require.NoError(t, err)
+		}
+	}
+	return tkn, claims
+}
+
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://localhost:18443", SharedSecret: "123456"})
+	sslPort := chooseRandomUnusedPort()
+	opts.SetCommon(CommonOpts{RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort), SharedSecret: "123456"})
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
-		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
-	require.Nil(t, err)
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=" + strconv.Itoa(port), "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
+		"--ssl.port=" + strconv.Itoa(sslPort), "--image.fs.path=/tmp"})
+	require.NoError(t, err)
+	defer os.Remove("/tmp/xyz")
+	defer os.Remove("/tmp/xyz/remark.db")
+	defer os.Remove("/tmp/ava-test.db")
 
 	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
+	app, err := opts.newServerApp(context.Background())
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(1 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPSServerStart(sslPort)
 
 	client := http.Client{
 		// prevent http redirect
@@ -173,23 +259,61 @@ func TestServerApp_WithSSL(t *testing.T) {
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		},
 	}
+	defer client.CloseIdleConnections()
 
 	// check http to https redirect response
-	resp, err := client.Get("http://localhost:18080/blah?param=1")
-	require.Nil(t, err)
+	resp, err := client.Get(fmt.Sprintf("http://localhost:%d/blah?param=1", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 307, resp.StatusCode)
-	assert.Equal(t, "https://localhost:18443/blah?param=1", resp.Header.Get("Location"))
+	assert.Equal(t, http.StatusTemporaryRedirect, resp.StatusCode)
+	assert.Equal(t, fmt.Sprintf("https://localhost:%d/blah?param=1", sslPort), resp.Header.Get("Location"))
 
 	// check https server
-	resp, err = client.Get("https://localhost:18443/ping")
-	require.Nil(t, err)
+	resp, err = client.Get(fmt.Sprintf("https://localhost:%d/ping", sslPort))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	cancel()
+	app.Wait()
+}
+
+func TestServerApp_WithRemote(t *testing.T) {
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	port := chooseRandomUnusedPort()
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
+		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+	require.NoError(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+	// create app
+	app, err := opts.newServerApp(context.Background())
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { _ = app.run(ctx) }()
+	waitForHTTPServerStart(port)
+
+	// send ping
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	defer http.DefaultClient.CloseIdleConnections()
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	cancel()
 	app.Wait()
 }
 
@@ -200,9 +324,9 @@ func TestServerApp_Failed(t *testing.T) {
 	p := flags.NewParser(&opts, flags.Default)
 
 	// RO bolt location
-	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
-	assert.Nil(t, err)
-	_, err = opts.newServerApp()
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
+	assert.NoError(t, err)
+	_, err = opts.newServerApp(context.Background())
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
 	t.Log(err)
 
@@ -211,9 +335,10 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
-	_, err = opts.newServerApp()
-	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
+	assert.NoError(t, err)
+	defer os.Remove("/tmp/remark.db")
+	_, err = opts.newServerApp(context.Background())
+	assert.EqualError(t, err, "failed to create backup store: can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
 	t.Log(err)
 
 	// invalid url
@@ -221,8 +346,8 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
-	_, err = opts.newServerApp()
+	assert.NoError(t, err)
+	_, err = opts.newServerApp(context.Background())
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
 
@@ -230,47 +355,182 @@ func TestServerApp_Failed(t *testing.T) {
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
-	assert.NotNil(t, err, "blah is invalid type")
+	assert.Error(t, err, "blah is invalid type")
 
 	opts.Store.Type = "blah"
-	_, err = opts.newServerApp()
+	_, err = opts.newServerApp(context.Background())
 	assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah")
 	t.Log(err)
+
+	// wrong redis location
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+	p = flags.NewParser(&opts, flags.Default)
+	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--cache.type=redis_pub_sub", "--cache.redis_addr=wrong_address"})
+	assert.NoError(t, err)
+	_, err = opts.newServerApp(context.Background())
+	assert.EqualError(t, err,
+		"failed to make cache: cache backend initialization, redis PubSub initialisation: "+
+			"problem subscribing to channel remark42-cache on address wrong_address: "+
+			"dial tcp: address wrong_address: missing port in address")
+	t.Log(err)
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = chooseRandomUnusedPort()
 		return o
 	})
+	time.AfterFunc(100*time.Millisecond, func() {
+		cancel()
+	})
 	st := time.Now()
 	err := app.run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err)
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100msec")
 	app.Wait()
 }
 
 func TestServerApp_MainSignal(t *testing.T) {
-
+	done := make(chan struct{})
 	go func() {
+		<-done
 		time.Sleep(250 * time.Millisecond)
 		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}()
-	st := time.Now()
 
 	s := ServerCommand{}
 	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
 	p := flags.NewParser(&s, flags.Default)
+	port := chooseRandomUnusedPort()
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=" + strconv.Itoa(port), "--image.fs.path=/tmp"}
+	defer os.Remove("/tmp/xyz")
+	defer os.Remove("/tmp/xyz/remark.db")
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	st := time.Now()
+	close(done)
 	err = s.Execute(args)
-	assert.NoError(t, err, "execute failed")
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	assert.NoError(t, err, "execute should be without errors")
+	assert.True(t, time.Since(st).Seconds() < 5, "should take under five sec", time.Since(st).Seconds())
+}
+
+func TestServerApp_DeprecatedArgs(t *testing.T) {
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{
+		"test",
+		"--notify.type=email",
+		"--notify.type=telegram",
+		"--notify.users=none",
+		"--notify.admins=none",
+		"--img-proxy",
+		"--notify.email.notify_admin",
+		"--auth.email.host=smtp.example.org",
+		"--auth.email.port=666",
+		"--auth.email.tls",
+		"--auth.email.user=test_user",
+		"--auth.email.passwd=test_password",
+		"--auth.email.timeout=15s",
+		"--auth.email.template=file.tmpl",
+		"--notify.telegram.token=abcd",
+		"--notify.telegram.timeout=3m",
+		"--notify.telegram.api=http://example.org",
+	}
+	assert.Empty(t, s.SMTP.Host)
+	assert.Empty(t, s.SMTP.Port)
+	assert.Empty(t, s.SMTP.TLS)
+	assert.Empty(t, s.SMTP.Username)
+	assert.Empty(t, s.SMTP.Password)
+	assert.Empty(t, s.SMTP.TimeOut)
+	_, err := p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlags := s.HandleDeprecatedFlags()
+	assert.ElementsMatch(t,
+		[]DeprecatedFlag{
+			{Old: "auth.email.host", New: "smtp.host", Version: "1.5"},
+			{Old: "auth.email.port", New: "smtp.port", Version: "1.5"},
+			{Old: "auth.email.tls", New: "smtp.tls", Version: "1.5"},
+			{Old: "auth.email.user", New: "smtp.username", Version: "1.5"},
+			{Old: "auth.email.passwd", New: "smtp.password", Version: "1.5"},
+			{Old: "auth.email.timeout", New: "smtp.timeout", Version: "1.5"},
+			{Old: "auth.email.template", Version: "1.5"},
+			{Old: "img-proxy", New: "image-proxy.http2https", Version: "1.5"},
+			{Old: "notify.email.notify_admin", New: "notify.admins=email", Version: "1.9"},
+			{Old: "notify.type", New: "notify.(users|admins)", Version: "1.9"},
+			{Old: "notify.telegram.token", New: "telegram.token", Version: "1.9"},
+			{Old: "notify.telegram.timeout", New: "telegram.timeout", Version: "1.9"},
+			{Old: "notify.telegram.api", Version: "1.9"},
+		},
+		deprecatedFlags)
+	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
+	assert.Equal(t, 666, s.SMTP.Port)
+	assert.Equal(t, true, s.SMTP.TLS)
+	assert.Equal(t, "test_user", s.SMTP.Username)
+	assert.Equal(t, "test_password", s.SMTP.Password)
+	assert.Equal(t, 15*time.Second, s.SMTP.TimeOut)
+}
+
+func TestServerApp_DeprecatedArgsCollisions(t *testing.T) {
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{
+		"test",
+		"--auth.email.host=smtp-old.example.org",
+		"--smtp.host=smtp-new.example.org",
+		"--auth.email.port=666",
+		"--smtp.port=999",
+		"--auth.email.user=test_user",
+		"--smtp.username=new_test_user",
+		"--auth.email.passwd=test_password",
+		"--smtp.password=new_test_password",
+		"--auth.email.timeout=15s",
+		"--smtp.timeout=20s",
+		"--notify.type=telegram",
+		"--notify.users=telegram",
+		"--notify.admins=none",
+		"--notify.telegram.token=abcd",
+		"--telegram.token=dcba",
+		"--notify.telegram.timeout=3m",
+		"--telegram.timeout=5m",
+	}
+	_, err := p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlagsCollisions := s.findDeprecatedFlagsCollisions()
+	assert.ElementsMatch(t,
+		[]DeprecatedFlag{
+			{Old: "notify.type", New: "notify.(users|admins)", Collision: true},
+			{Old: "auth.email.host", New: "smtp.host", Collision: true},
+			{Old: "auth.email.port", New: "smtp.port", Collision: true},
+			{Old: "auth.email.user", New: "smtp.username", Collision: true},
+			{Old: "auth.email.passwd", New: "smtp.password", Collision: true},
+			{Old: "auth.email.timeout", New: "smtp.timeout", Collision: true},
+			{Old: "notify.telegram.token", New: "telegram.token", Collision: true},
+			{Old: "notify.telegram.timeout", New: "telegram.timeout", Collision: true},
+		},
+		deprecatedFlagsCollisions)
+
+	// case which should return nothing
+	s = ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+	p = flags.NewParser(&s, flags.Default)
+	args = []string{
+		"test",
+		"--auth.email.host=smtp-old.example.org",
+		"--smtp.host=''",
+	}
+	_, err = p.ParseArgs(args)
+	require.NoError(t, err)
+	deprecatedFlagsCollisions = s.findDeprecatedFlagsCollisions()
+	assert.Empty(t, []DeprecatedFlag{}, deprecatedFlagsCollisions)
 }
 
 func Test_ACMEEmail(t *testing.T) {
@@ -279,9 +539,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p := flags.NewParser(&cmd, flags.Default)
 	args := []string{"--ssl.type=auto"}
 	_, err := p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err := cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -289,9 +549,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--ssl.acme-email=adminname@adminhost.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "adminname@adminhost.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -299,9 +559,9 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared", "--admin.shared.email=superadmin@admin.com"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "superadmin@admin.com", cfg.ACMEEmail)
 
 	cmd = ServerCommand{}
@@ -309,20 +569,21 @@ func Test_ACMEEmail(t *testing.T) {
 	p = flags.NewParser(&cmd, flags.Default)
 	args = []string{"--ssl.type=auto", "--admin.type=shared"}
 	_, err = p.ParseArgs(args)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	cfg, err = cmd.makeSSLConfig()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := chooseRandomUnusedPort()
+	app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	waitForHTTPServerStart(port)
 
 	// make a token for user dev
 	tkService := app.restSrv.Authenticator.TokenService()
@@ -344,76 +605,211 @@ func TestServerAuthHooks(t *testing.T) {
 	require.NoError(t, err)
 	t.Log(tk)
 
+	client := http.Client{Timeout: 10 * time.Second}
+	defer client.CloseIdleConnections()
+
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
-	defer resp.Body.Close()
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
+	// add comment with no-aud claim
+	claimsNoAud := claims
+	claimsNoAud.Audience = ""
+	tkNoAud, err := tkService.Token(claimsNoAud)
+	require.NoError(t, err)
+	t.Logf("no-aud claims: %s", tkNoAud)
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/",
+	"site": "remark"}}`))
+	require.NoError(t, err)
+	req.Header.Set("X-JWT", tkNoAud)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
+
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
-	assert.Nil(t, e)
+	req, err = http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), http.NoBody)
+	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, e = client.Do(req)
-	require.Nil(t, e)
-	defer resp.Body.Close()
+	resp, err = client.Do(req)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
-	b, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
+	b, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
 	t.Log(string(b))
 
-	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
-
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err = client.Do(req)
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
+		"blocked user can't post, \n"+tk+"\n"+string(body))
 
+	cancel()
 	app.Wait()
+	client.CloseIdleConnections()
 }
 
-func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
+func TestServerCommand_parseSameSite(t *testing.T) {
+	tbl := []struct {
+		inp string
+		res http.SameSite
+	}{
+		{"", http.SameSiteDefaultMode},
+		{"default", http.SameSiteDefaultMode},
+		{"blah", http.SameSiteDefaultMode},
+		{"none", http.SameSiteNoneMode},
+		{"lax", http.SameSiteLaxMode},
+		{"strict", http.SameSiteStrictMode},
+	}
+
+	cmd := ServerCommand{}
+	for i, tt := range tbl {
+		tt := tt
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.res, cmd.parseSameSite(tt.inp))
+		})
+	}
+}
+
+func Test_splitAtCommas(t *testing.T) {
+	tbl := []struct {
+		inp string
+		res []string
+	}{
+		{"a string", []string{"a string"}},
+		{"vv1, vv2, vv3", []string{"vv1", "vv2", "vv3"}},
+		{`"vv1, blah", vv2, vv3`, []string{"vv1, blah", "vv2", "vv3"}},
+		{
+			`Access-Control-Allow-Headers:"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type",header123:val, foo:"bar1,bar2"`,
+			[]string{"Access-Control-Allow-Headers:\"DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type\"", "header123:val", "foo:\"bar1,bar2\""},
+		},
+		{"", []string{}},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.res, splitAtCommas(tt.inp))
+		})
+	}
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	defer client.CloseIdleConnections()
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func waitForHTTPSServerStart(port int) {
+	// wait for up to 3 seconds for HTTPS server to start
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
+		if conn != nil {
+			_ = conn.Close()
+			break
+		}
+	}
+}
+
+func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
 	cmd := ServerCommand{}
 	cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "secret"})
 
 	// prepare options
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
-	require.Nil(t, err)
-	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	require.NoError(t, err)
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp/remark42_test", "fs", "/tmp/remark42_test", "/tmp/remark42_test"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
 	cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
+	cmd.Auth.Microsoft.CSEC, cmd.Auth.Microsoft.CID = "csec", "cid"
+	cmd.Auth.Twitter.CSEC, cmd.Auth.Twitter.CID = "csec", "cid"
+	cmd.Auth.Patreon.CSEC, cmd.Auth.Patreon.CID = "csec", "cid"
+	cmd.Auth.Telegram = true
+	cmd.Telegram.Token = "token"
+	cmd.Auth.Email.Enable = true
+	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
 	cmd.BackupLocation = "/tmp"
-	cmd.Notify.Type = "telegram"
-	cmd.Notify.Telegram.API = "http://127.0.0.1:12340/"
-	cmd.Notify.Telegram.Token = "blah"
+	cmd.Notify.Users = []string{"email"}
+	cmd.Notify.Admins = []string{"email"}
+	cmd.Notify.Email.From = "from@example.org"
+	cmd.Notify.Email.VerificationSubject = "test verification email subject"
+	cmd.SMTP.Host = "127.0.0.1"
+	cmd.SMTP.Port = 25
+	cmd.SMTP.Username = "test_user"
+	cmd.SMTP.Password = "test_password"
+	cmd.SMTP.TimeOut = time.Second
+	cmd.UpdateLimit = 10
+	cmd.Admin.Type = "shared"
+	cmd.Admin.Shared.Admins = []string{"id1", "id2"}
+	cmd.RestrictedNames = []string{"umputun", "bobuk"}
+	cmd.emailMsgTemplatePath = "../../templates/email_reply.html.tmpl"
+	cmd.emailVerificationTemplatePath = "../../templates/email_confirmation_subscription.html.tmpl"
 	cmd = fn(cmd)
 
-	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
+	app, ctx, cancel := createAppFromCmd(t, cmd)
 
-	// create app
-	app, err := cmd.newServerApp()
-	require.Nil(t, err)
-
-	ctx, cancel := context.WithCancel(context.Background())
+	// cleanup the remark.db file after context is canceled
 	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
+		<-ctx.Done()
+		os.RemoveAll(cmd.Store.Bolt.Path)
+		os.RemoveAll(cmd.Avatar.FS.Path)
+
 	}()
-	return app, ctx
+
+	return app, ctx, cancel
+}
+
+func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(context.Background())
+	app, err := cmd.newServerApp(ctx)
+	require.NoError(t, err)
+
+	rand.Seed(time.Now().UnixNano())
+	return app, ctx, cancel
+}
+
+func TestMain(m *testing.M) {
+	// ignore is added only for GitHub Actions, can't reproduce locally
+	goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
 }

backend/app/cmd/testdata/email.tmpl

@@ -0,0 +1 @@
+The token is {{.Token}}

backend/app/cmd/testdata/remap_urls.txt

@@ -0,0 +1,2 @@
+http://oldsite.com* https://newsite.com*
+http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1

backend/app/cmd/web/index.html

@@ -0,0 +1 @@
+This stub page would be replaced by the frontend statically built HTML during the Docker image build.

backend/app/main.go

@@ -8,9 +8,9 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
-	"github.com/umputun/remark/backend/app/cmd"
+	"github.com/umputun/remark42/backend/app/cmd"
 )
 
 // Opts with all cli commands and flags
@@ -21,9 +21,10 @@ type Opts struct {
 	RestoreCmd cmd.RestoreCommand `command:"restore"`
 	AvatarCmd  cmd.AvatarCommand  `command:"avatar"`
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
 	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
-	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
+	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"the shared secret key used to sign JWT, should be a random, long, hard-to-guess string"`
 
 	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
 }
@@ -44,6 +45,7 @@ func main() {
 			SharedSecret: opts.SharedSecret,
 			Revision:     revision,
 		})
+		logDeprecatedParams(c.HandleDeprecatedFlags())
 		err := c.Execute(args)
 		if err != nil {
 			log.Printf("[ERROR] failed with %+v", err)
@@ -62,12 +64,28 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 		return
 	}
 	log.Setup(log.Msec, log.LevelBraces)
 }
 
+// logs usual and "collision" deprecated parameters
+func logDeprecatedParams(params []cmd.DeprecatedFlag) {
+	for _, entry := range params {
+		var deprecationNote string
+		if entry.Collision {
+			deprecationNote = fmt.Sprintf("[ERROR] deprecated --%s and new --%s options are set to different values, old one is ignored: please remove it", entry.Old, entry.New)
+		} else {
+			deprecationNote = fmt.Sprintf("[WARN] --%s is deprecated since v%s and will be removed in the future", entry.Old, entry.Version)
+			if entry.New != "" {
+				deprecationNote += fmt.Sprintf(", please use --%s instead", entry.New)
+			}
+		}
+		log.Print(deprecationNote)
+	}
+}
+
 // getDump reads runtime stack and returns as a string
 func getDump() string {
 	maxSize := 5 * 1024 * 1024
@@ -79,9 +97,10 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
+// nolint:gochecknoinits // can't avoid it in this place
 func init() {
 	// catch SIGQUIT and print stack traces
-	sigChan := make(chan os.Signal)
+	sigChan := make(chan os.Signal, 1)
 	go func() {
 		for range sigChan {
 			log.Printf("[INFO] SIGQUIT detected, dump:\n%s", getDump())

backend/app/main_test.go

@@ -1,53 +1,122 @@
 package main
 
 import (
-	"io/ioutil"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
-	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
+	dir, err := os.MkdirTemp(os.TempDir(), "remark42")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	port := chooseRandomUnusedPort()
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
+	done := make(chan struct{})
 	go func() {
-		time.Sleep(500 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		<-done
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.NoError(t, e)
 	}()
 
-	wg := sync.WaitGroup{}
-	wg.Add(1)
+	finished := make(chan struct{})
 	go func() {
-		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-		wg.Done()
+		close(finished)
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	// defer cleanup because require check below can fail
+	defer func() {
+		close(done)
+		<-finished
+	}()
 
-	// send ping
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
-	require.Nil(t, err)
+	waitForHTTPServerStart(port)
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.NoError(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
 	assert.Equal(t, "pong", string(body))
+}
 
-	wg.Wait()
+func TestMain_WithWebhook(t *testing.T) {
+	dir, err := os.MkdirTemp(os.TempDir(), "remark42")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	var webhookSent int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.StoreInt32(&webhookSent, 1)
+		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+
+		b, e := io.ReadAll(r.Body)
+		defer r.Body.Close()
+
+		assert.Nil(t, e)
+		assert.Equal(t, "Comment: env test", string(b))
+	}))
+	defer ts.Close()
+
+	port := chooseRandomUnusedPort()
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg",
+		"--admin-passwd=password", "--site=remark", "--notify.admins=webhook"}
+
+	err = os.Setenv("NOTIFY_WEBHOOK_URL", ts.URL)
+	assert.NoError(t, err)
+	err = os.Setenv("NOTIFY_WEBHOOK_TEMPLATE", "Comment: {{.Orig}}")
+	assert.NoError(t, err)
+	err = os.Setenv("NOTIFY_WEBHOOK_HEADERS", "Content-Type:application/json")
+	assert.NoError(t, err)
+
+	done := make(chan struct{})
+	go func() {
+		<-done
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.NoError(t, e)
+	}()
+
+	finished := make(chan struct{})
+	go func() {
+		main()
+		assert.Eventually(t, func() bool {
+			return atomic.LoadInt32(&webhookSent) == int32(1)
+		}, time.Second, 100*time.Millisecond, "webhook was not sent")
+		close(finished)
+	}()
+
+	// defer cleanup because require check below can fail
+	defer func() {
+		close(done)
+		<-finished
+	}()
+
+	waitForHTTPServerStart(port)
+
+	resp, err := http.Post(fmt.Sprintf("http://admin:password@localhost:%d/api/v1/comment", port), "",
+		strings.NewReader(`{"text": "env test", "locator":{"url": "https://radio-t.com", "site": "remark"}}`))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
 func TestGetDump(t *testing.T) {
@@ -55,5 +124,38 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "goroutine"))
 	assert.True(t, strings.Contains(dump, "[running]"))
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
-	log.Printf("\n dump: %s", dump)
+	t.Logf("\n dump: %s", dump)
+}
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 10 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	defer client.CloseIdleConnections()
+	for i := 0; i < 100; i++ {
+		time.Sleep(time.Millisecond * 100)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	// both ignores are for leaks which are detected locally
+	goleak.VerifyTestMain(
+		m,
+		goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
+		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+	)
 }

backend/app/migrator/backup.go

@@ -4,14 +4,12 @@ import (
 	"compress/gzip"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 )
 
 // AutoBackup struct handles daily backups params for siteID
@@ -27,6 +25,7 @@ type AutoBackup struct {
 func (ab AutoBackup) Do(ctx context.Context) {
 	log.Printf("[INFO] activate auto-backup for %s under %s, duration %s", ab.SiteID, ab.BackupLocation, ab.Duration)
 	tick := time.NewTicker(ab.Duration)
+	defer tick.Stop()
 	log.Printf("[DEBUG] first backup for %s at %s", ab.SiteID, time.Now().Add(ab.Duration))
 
 	for {
@@ -48,35 +47,40 @@ func (ab AutoBackup) Do(ctx context.Context) {
 func (ab AutoBackup) makeBackup() (string, error) {
 	log.Printf("[DEBUG] make backup for %s", ab.SiteID)
 	backupFile := fmt.Sprintf("%s/backup-%s-%s.gz", ab.BackupLocation, ab.SiteID, time.Now().Format("20060102"))
-	fh, err := os.Create(backupFile)
+	fh, err := os.Create(backupFile) //nolint:gosec // harmless
 	if err != nil {
-		return "", errors.Wrapf(err, "can't create backup file %s", backupFile)
+		return "", fmt.Errorf("can't create backup file %s: %w", backupFile, err)
 	}
 	gz := gzip.NewWriter(fh)
 
 	if _, err = ab.Exporter.Export(gz, ab.SiteID); err != nil {
-		return "", errors.Wrapf(err, "export failed for %s", ab.SiteID)
+		return "", fmt.Errorf("export failed for %s: %w", ab.SiteID, err)
 	}
 	if err = gz.Close(); err != nil {
-		return "", errors.Wrapf(err, "can't close gz for %s", backupFile)
+		return "", fmt.Errorf("can't close gz for %s: %w", backupFile, err)
 	}
 	if err = fh.Close(); err != nil {
-		return "", errors.Wrapf(err, "can't close file handler for %s", backupFile)
+		return "", fmt.Errorf("can't close file handler for %s: %w", backupFile, err)
 	}
 	log.Printf("[DEBUG] created backup file %s", backupFile)
 	return backupFile, nil
 }
 
 func (ab AutoBackup) removeOldBackupFiles() {
-	files, err := ioutil.ReadDir(ab.BackupLocation)
+	files, err := os.ReadDir(ab.BackupLocation)
 	if err != nil {
 		log.Printf("[WARN] can't read files in backup directory %s, %s", ab.BackupLocation, err)
 		return
 	}
 	backFiles := []os.FileInfo{}
 	for _, file := range files {
+		info, e := file.Info()
+		if e != nil {
+			log.Printf("[WARN] can't read info for directory %s, %s", file.Name(), e)
+			return
+		}
 		if strings.HasPrefix(file.Name(), "backup-"+ab.SiteID) {
-			backFiles = append(backFiles, file)
+			backFiles = append(backFiles, info)
 		}
 	}
 	sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })

backend/app/migrator/backup_test.go

@@ -4,33 +4,34 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0o700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
-		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-		assert.Nil(t, err)
+		err := os.WriteFile(fname, []byte("blah"), 0o600)
+		assert.NoError(t, err)
 	}
 	fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
-	err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-	assert.Nil(t, err)
+	err := os.WriteFile(fname, []byte("blah"), 0o600)
+	assert.NoError(t, err)
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
 	bk.removeOldBackupFiles()
-	ff, err := ioutil.ReadDir(loc)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
+	ff, err := os.ReadDir(loc)
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
 	assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
 	assert.Equal(t, "backup-site1-20171209.gz", ff[1].Name())
 	assert.Equal(t, "backup-site1-20171210.gz", ff[2].Name())
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0o700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0o700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
@@ -75,7 +76,7 @@ func TestBackup_Do(t *testing.T) {
 
 type mockExporter struct{}
 
-func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
+func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
 	_, err := w.Write([]byte("some export blah blah 1234567890"))
 	return 1000, err
 }

backend/app/migrator/commento.go

@@ -0,0 +1,133 @@
+package migrator
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/umputun/remark42/backend/app/store"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// Commento implements Importer from commento export json
+type Commento struct {
+	DataStore Store
+}
+
+// Credit: https://gitlab.com/commento/commento/-/blob/master/api/domain_import_commento.go#L11-L15
+type commentoExport struct {
+	Version    int                 `json:"version"`
+	Comments   []commentoComment   `json:"comments"`
+	Commenters []commentoCommenter `json:"commenters"`
+}
+
+// Credit: https://gitlab.com/commento/commento/-/blob/master/api/comment.go#L7-L20
+type commentoComment struct {
+	CommentHex   string    `json:"commentHex"`
+	Domain       string    `json:"domain,omitempty"`
+	Path         string    `json:"url,omitempty"`
+	CommenterHex string    `json:"commenterHex"`
+	Markdown     string    `json:"markdown"`
+	HTML         string    `json:"html"`
+	ParentHex    string    `json:"parentHex"`
+	Score        int       `json:"score"`
+	State        string    `json:"state,omitempty"`
+	CreationDate time.Time `json:"creationDate"`
+	Direction    int       `json:"direction"`
+	Deleted      bool      `json:"deleted"`
+}
+
+// Credit: https://gitlab.com/commento/commento/-/blob/master/api/commenter.go#L7-L16
+type commentoCommenter struct {
+	CommenterHex string    `json:"commenterHex,omitempty"`
+	Email        string    `json:"email,omitempty"`
+	Name         string    `json:"name"`
+	Link         string    `json:"link"`
+	Photo        string    `json:"photo"`
+	Provider     string    `json:"provider,omitempty"`
+	JoinDate     time.Time `json:"joinDate,omitempty"`
+	IsModerator  bool      `json:"isModerator"`
+}
+
+// Import comments from Commento and save to store
+func (d *Commento) Import(r io.Reader, siteID string) (size int, err error) {
+	if e := d.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
+	}
+
+	commentsCh := d.convert(r, siteID)
+	failed, passed := 0, 0
+	for c := range commentsCh {
+		if _, err = d.DataStore.Create(c); err != nil {
+			failed++
+			continue
+		}
+		passed++
+	}
+
+	if failed > 0 {
+		err = fmt.Errorf("failed to save %d comments", failed)
+		if passed == 0 {
+			err = fmt.Errorf("import failed")
+		}
+	}
+
+	log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
+
+	return passed, err
+}
+
+func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
+	commentsCh := make(chan store.Comment)
+
+	decoder := json.NewDecoder(r)
+
+	go func() {
+		var exportedData commentoExport
+		err := decoder.Decode(&exportedData)
+		if err != nil {
+			log.Printf("[WARN] can't decode commento export json, %s", err.Error())
+		}
+
+		usersMap := map[string]store.User{}
+		for _, commenter := range exportedData.Commenters {
+			usersMap[commenter.CommenterHex] = store.User{
+				Name:    commenter.Name,
+				ID:      "commento_" + store.EncodeID(commenter.CommenterHex),
+				Picture: commenter.Photo,
+			}
+		}
+
+		for _, comment := range exportedData.Comments {
+			u, ok := usersMap[comment.CommenterHex]
+			if !ok {
+				continue
+			}
+
+			if comment.Deleted {
+				continue
+			}
+
+			c := store.Comment{
+				ID: comment.CommentHex,
+				Locator: store.Locator{
+					URL:    comment.Path,
+					SiteID: siteID,
+				},
+				User:      u,
+				Text:      comment.Markdown,
+				Timestamp: comment.CreationDate,
+				ParentID:  comment.ParentHex,
+				Imported:  true,
+			}
+
+			commentsCh <- c
+		}
+
+		close(commentsCh)
+	}()
+
+	return commentsCh
+}

backend/app/migrator/commento_test.go

@@ -0,0 +1,54 @@
+package migrator
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
+)
+
+func TestCommento_Import(t *testing.T) {
+	defer os.Remove("/tmp/remark-test.db")
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
+
+	d := Commento{DataStore: &dataStore}
+	fh, err := os.Open("testdata/commento.json")
+	require.NoError(t, err)
+	size, err := d.Import(fh, "test")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(last), "2 comments imported")
+
+	t.Log(last[0])
+
+	c := last[0] // last reverses, get first one
+	assert.Equal(t, "Great reply!", c.Text)
+	assert.Equal(t, "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9", c.ID)
+	assert.Equal(t, "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", c.ParentID)
+	assert.Equal(t, store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"}, c.Locator)
+	assert.Equal(t, "Saturnin Uf", c.User.Name)
+	assert.Equal(t, "commento_35369aeb6ac5255de30410a0f86dc71eb9c6d0ca", c.User.ID)
+	assert.True(t, c.Imported)
+
+	posts, err := dataStore.List("test", 0, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(posts), "1 post")
+
+	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, count)
+}

backend/app/migrator/disqus.go

@@ -2,14 +2,14 @@ package migrator
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"strings"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // Disqus implements Importer from disqus xml
@@ -44,6 +44,7 @@ type disqusComment struct {
 	Tid            uid       `xml:"thread"`
 	Pid            uid       `xml:"parent"`
 	IsSpam         bool      `xml:"isSpam"`
+	Deleted        bool      `xml:"isDeleted"`
 }
 
 type uid struct {
@@ -52,9 +53,8 @@ type uid struct {
 
 // Import from disqus and save to store
 func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
-
-	if err = d.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := d.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
 	commentsCh := d.convert(r, siteID)
@@ -68,9 +68,9 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
 	}
 
 	if failed > 0 {
-		err = errors.Errorf("failed to save %d comments", failed)
+		err = fmt.Errorf("failed to save %d comments", failed)
 		if passed == 0 {
-			err = errors.New("import failed")
+			err = fmt.Errorf("import failed")
 		}
 	}
 
@@ -82,15 +82,15 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
 // convert disqus stream (xml) from reader and fill channel of comments.
 // runs async and closes channel on completion.
 func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
-
 	postsMap := map[string]string{} // tid:url
 	decoder := xml.NewDecoder(r)
 	commentsCh := make(chan store.Comment)
 
 	stats := struct {
-		inpThreads, inpComments     int
-		commentsCount, spamComments int
-		failedThreads, failedPosts  int
+		inpThreads, inpComments          int
+		commentsCount, spamComments      int
+		failedThreads, failedPosts       int
+		deletedComments, skippedComments int
 	}{}
 
 	go func() {
@@ -100,34 +100,48 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				break
 			}
 
-			switch se := t.(type) {
-			case xml.StartElement:
+			if se, ok := t.(xml.StartElement); ok {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
 					}
+					if thread.Deleted {
+						continue
+					}
 					postsMap[thread.UID] = thread.Link
 					continue
 				}
+
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue
 					}
+					if comment.Deleted {
+						stats.deletedComments++
+						continue
+					}
 					if comment.IsSpam {
 						stats.spamComments++
 						continue
 					}
+
+					url, ok := postsMap[comment.Tid.Val]
+					if !ok {
+						stats.skippedComments++
+						continue
+					}
+
 					c := store.Comment{
 						ID:      comment.UID,
-						Locator: store.Locator{URL: postsMap[comment.Tid.Val], SiteID: siteID},
+						Locator: store.Locator{URL: url, SiteID: siteID},
 						User: store.User{
 							ID:   "disqus_" + store.EncodeID(comment.AuthorUserName),
 							Name: comment.AuthorName,
@@ -136,9 +150,10 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 						Text:      d.cleanText(comment.Message),
 						Timestamp: comment.CreatedAt,
 						ParentID:  comment.Pid.Val,
+						Imported:  true,
 					}
-					if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
-						c.User.ID = "disqus_" + c.User.Name
+					if comment.AuthorUserName == "" { // empty comment.AuthorUserName from disqus
+						c.User.ID = "disqus_" + store.EncodeID(c.User.Name)
 					}
 					if c.ID == "" { // no comment.UID
 						c.ID = comment.ID
@@ -159,6 +174,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 }
 
 func (*Disqus) cleanText(text string) string {
+	text = strings.TrimSpace(text)
 	text = strings.Replace(text, "\n", "", -1)
 	text = strings.Replace(text, "\t", "", -1)
 	return text

backend/app/migrator/disqus_test.go

@@ -6,29 +6,32 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	d := Disqus{DataStore: &dataStore}
-	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
-	assert.Nil(t, err)
+	fh, err := os.Open("testdata/disqus.xml")
+	require.NoError(t, err)
+	size, err := d.Import(fh, "test")
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(last), "4 comments imported")
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 4, len(last), "4 comments imported")
 
 	c := last[len(last)-1] // last reverses, get first one
 	assert.True(t, strings.HasPrefix(c.Text, "<p>The quick brown fox"))
@@ -38,25 +41,92 @@ func TestDisqus_Import(t *testing.T) {
 	assert.Equal(t, "Alexander Blah", c.User.Name)
 	assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID)
 	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
+	assert.True(t, c.Imported)
+
+	c = last[1] // get comment with empty username
+	assert.Equal(t, "No Username", c.User.Name)
+	assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
 
 	posts, err := dataStore.List("test", 0, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(posts), "2 posts")
 
 	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, count)
 }
 
+func TestDisqus_ImportDeletedThread(t *testing.T) {
+	defer os.Remove("/tmp/remark-test.db")
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
+	d := Disqus{DataStore: &dataStore}
+	fh, err := os.Open("testdata/disqus-deleted-thread.xml")
+	require.NoError(t, err)
+	size, err := d.Import(fh, "test")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(last), "2 comments imported")
+
+	c := last[len(last)-1] // last reverses, get first one
+	assert.True(t, strings.HasPrefix(c.Text, "<p>Google App Engine "), c.Text)
+	assert.Equal(t, "299986072", c.ID)
+	assert.Equal(t, "", c.ParentID)
+	assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator)
+	assert.Equal(t, "No Username", c.User.Name)
+	assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
+	assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP)
+	assert.True(t, c.Imported)
+
+	c = last[1] // get comment with empty username
+	assert.Equal(t, "No Username", c.User.Name)
+	assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
+}
+
+func TestDisqus_ImportDeletedPost(t *testing.T) {
+	defer os.Remove("/tmp/remark-test.db")
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.NoError(t, err, "create store")
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
+	d := Disqus{DataStore: &dataStore}
+	fh, err := os.Open("testdata/disqus-deleted-post.xml")
+	require.NoError(t, err)
+	size, err := d.Import(fh, "test")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, size, "1 post deleted")
+
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
+
+	c := last[len(last)-1] // last reverses, get first one
+	assert.True(t, strings.HasPrefix(c.Text, "<p>Microsoft "), c.Text)
+	assert.Equal(t, "299744309", c.ID)
+	assert.Equal(t, "", c.ParentID)
+	assert.Equal(t, store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"}, c.Locator)
+	assert.Equal(t, "mikhail", c.User.Name)
+	assert.Equal(t, "disqus_1b6709749c0cab163db9070cc4edf3322b398d8c", c.User.ID)
+	assert.Equal(t, "9d3657a95a4e341510404bd8bf1a363faefd4ba4", c.User.IP)
+	assert.True(t, c.Imported)
+}
+
 func TestDisqus_Convert(t *testing.T) {
 	d := Disqus{}
-	ch := d.convert(strings.NewReader(xmlTestDisqus), "test")
+	fh, err := os.Open("testdata/disqus.xml")
+	require.NoError(t, err)
+	ch := d.convert(fh, "test")
 
 	res := []store.Comment{}
 	for comment := range ch {
 		res = append(res, comment)
 	}
-	assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
+	require.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
 
 	exp0 := store.Comment{
 		ID: "299619020",
@@ -70,160 +140,8 @@ func TestDisqus_Convert(t *testing.T) {
 			ID:   "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434",
 			IP:   "178.178.178.178",
 		},
+		Imported: true,
 	}
 	exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z")
 	assert.Equal(t, exp0, res[0])
 }
-
-var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
-<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
-
-	<category dsq:id="707279">
-		<forum>radiot</forum>
-		<title>General</title>
-		<isDefault>true</isDefault>
-	</category>
-
-	<thread dsq:id="247918464">
-		<id/>
-		<forum>radiot</forum>
-		<category dsq:id="707279"/>
-		<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
-		<title>Радио-Т: Радио-Т 229</title>
-		<message/>
-		<createdAt>2011-03-07T20:46:25Z</createdAt>
-		<author>
-			<email>umputun@gmail.com</email>
-			<name>Umputun</name>
-			<isAnonymous>false</isAnonymous>
-			<username>umputun</username>
-		</author>
-		<ipAddress>98.212.28.115</ipAddress>
-		<isClosed>false</isClosed>
-		<isDeleted>false</isDeleted>
-	</thread>
-
-	<thread dsq:id="247937687">
-		<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
-		<forum>radiot</forum>
-		<category dsq:id="707279"/>
-		<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
-		<title>Радио-Т: Радио-Т 229</title>
-		<message/>
-		<createdAt>2011-03-07T21:17:17Z</createdAt>
-		<author>
-			<email>umputun@gmail.com</email>
-			<name>Umputun</name>
-			<isAnonymous>false</isAnonymous>
-			<username>umputun</username>
-		</author>
-		<ipAddress>80.250.214.235</ipAddress>
-		<isClosed>true</isClosed>
-		<isDeleted>false</isDeleted>
-	</thread>
-
-
-	<post dsq:id="299619020">
-		<id>3565798471341011339</id>
-		<message>
-			<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
-		</message>
-		<createdAt>2011-08-31T15:16:29Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email/>
-			<name>Alexander Blah</name>
-			<isAnonymous>false</isAnonymous>
-			<username>facebook-1787732238</username>
-		</author>
-		<ipAddress>178.178.178.178</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-
-	<post dsq:id="299744309">
-		<id>3029154520436241933</id>
-		<message>
-			<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
-		</message>
-		<createdAt>2011-08-31T17:44:22Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email>mihail.noname@gmail.com</email>
-			<name>mikhail</name>
-			<isAnonymous>false</isAnonymous>
-			<username>mikhail-noname</username>
-		</author>
-		<ipAddress>195.195.195.139</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-
-	<post dsq:id="299986072">
-		<id>6580890074280459209</id>
-		<message>
-			<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
-		</message>
-		<createdAt>2011-08-31T22:48:43Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email>dmitri.noname@gmail.com</email>
-			<name>Dmitry Noname</name>
-			<isAnonymous>false</isAnonymous>
-			<username>google-74b9e7568ef6860e93862c5d77590123</username>
-		</author>
-		<ipAddress>89.89.89.139</ipAddress>
-		<thread dsq:id="247918464"/>
-	</post>
-
-	<post>
-		<id>12345678890</id>
-		<message>This comment had no ID</message>
-		<createdAt>2011-08-31T22:49:43Z</createdAt>
-		<forum>radiot</forum>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email>blah.noname@gmail.com</email>
-			<name>Blah Noname</name>
-			<isAnonymous>false</isAnonymous>
-			<username>74b9e7568ef6860e93862c5d77590123</username>
-		</author>
-		<ipAddress>189.89.89.139</ipAddress>
-		<thread dsq:id="247918464"/>
-	</post>
-
-	<post dsq:id="299986073">
-		<id>6580890074280459219</id>
-		<message>some ugly spam</message>
-		<createdAt>2011-09-30T22:48:43Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>true</isSpam>
-		<author>
-			<email>spam.noname@gmail.com</email>
-			<name>Spam Noname</name>
-			<isAnonymous>false</isAnonymous>
-			<username>google-2c5d77590123</username>
-		</author>
-		<ipAddress>189.89.89.139</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-
-	<post dsq:id="x299986073">
-		<message>some bad comment</message>
-		<createdAt>2011-x09-30T22:48:43Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>123</isSpam>
-		<author>
-			<email>noname@gmail.com</email>
-			<name>Noname</name>
-			<isAnonymous>true</isAnonymous>
-			<username>google-2c5d77590123</username>
-		</author>
-		<ipAddress>189.89.89.39</ipAddress>
-		<thread dsq:id=247937687/>
-	</post>
-
-</disqus>
-`

backend/app/migrator/mapper.go

@@ -0,0 +1,73 @@
+package migrator
+
+import (
+	"fmt"
+	"io"
+	"strings"
+)
+
+// URLMapper implements Mapper interface
+type URLMapper struct {
+	rules map[string]string
+}
+
+// NewURLMapper reads rules from given reader and returns initialized URLMapper
+// if given rules are valid.
+func NewURLMapper(reader io.Reader) (Mapper, error) {
+	u := &URLMapper{}
+	if err := u.loadRules(reader); err != nil {
+		return u, err
+	}
+	return u, nil
+}
+
+// loadRules loads url-mapping rules from reader to mapper.
+// Rules must be a text consists of rows separated by \n.
+// Each row holds from-url and to-url separated by space.
+// If urls end with asterisk (*) it means try to match by prefix.
+// Example:
+// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
+// https://www.myblog.com/* https://myblog.com/*
+func (u *URLMapper) loadRules(reader io.Reader) error {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	rulesText := strings.TrimSpace(string(data))
+
+	u.rules = make(map[string]string)
+
+	for _, row := range strings.Split(rulesText, "\n") {
+		row = strings.TrimSpace(row)
+		urls := strings.Split(row, " ")
+		if len(urls) != 2 {
+			return fmt.Errorf("bad row %s", row)
+		}
+
+		from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
+		u.rules[from] = to
+	}
+	return nil
+}
+
+// URL maps given url to another url according loaded url-rules.
+// If not matched returns given url.
+func (u *URLMapper) URL(url string) string {
+	if newURL, ok := u.rules[url]; ok {
+		return newURL
+	}
+	// try to match by prefix
+	for oldURL, newURL := range u.rules {
+		if !strings.HasSuffix(oldURL, "*") {
+			continue
+		}
+		oldURL = strings.TrimSuffix(oldURL, "*")
+		newURL = strings.TrimSuffix(newURL, "*")
+		if strings.HasPrefix(url, oldURL) {
+			return newURL + strings.TrimPrefix(url, oldURL)
+		}
+	}
+	// search failed, return given url
+	return url
+}

backend/app/migrator/mapper_test.go

@@ -0,0 +1,90 @@
+package migrator
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUrlMapper_URL(t *testing.T) {
+	// want remap urls from https://radio-t.com to https://www.radio-t.com
+	// also map individual urls
+	rules := strings.NewReader(`
+https://radio-t.com* https://www.radio-t.com*
+https://radio-t.com/p/2018/09/22////podcast-616/ https://www.radio-t.com/p/2018/09/22/podcast-616/
+https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/
+`)
+
+	mapper, err := NewURLMapper(rules)
+	assert.NoError(t, err)
+
+	// if url not matched mapper should return given url
+	assert.Equal(t, "https://any.com/post/1/", mapper.URL("https://any.com/post/1/"))
+	assert.Equal(t, "https://radio-t.co", mapper.URL("https://radio-t.co"))
+	// check strict matching
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22////podcast-616/"))
+	assert.Equal(t, "https://www.radio-t.com/p/2018/09/22/podcast-616/", mapper.URL("https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1"))
+	// check pattern matching (by prefix)
+	assert.Equal(t, "https://www.radio-t.com/p/post/123/", mapper.URL("https://radio-t.com/p/post/123/"))
+
+	// want remap from http to https
+	rules = strings.NewReader(`http://anysite.com/p/123 https://anysite.com/p/321`)
+	mapper, err = NewURLMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/321", mapper.URL("http://anysite.com/p/123"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("https://anysite.com/")) // not exist
+
+	// want remap from http to https by pattern
+	rules = strings.NewReader(`http://anysite.com* https://anysite.com*`)
+	mapper, err = NewURLMapper(rules)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://anysite.com/p/1", mapper.URL("http://anysite.com/p/1"))
+	assert.Equal(t, "https://anysite.com/", mapper.URL("http://anysite.com/"))
+	assert.Equal(t, "https://notexist", mapper.URL("https://notexist"))
+}
+
+func TestUrlMapper_New(t *testing.T) {
+	cases := []struct {
+		rules       string
+		expectError bool
+	}{
+		// bad input, expect error
+		{
+			rules:       "https://radio-t.com ",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com https://radio-t.com\n https://radio-t.com",
+			expectError: true,
+		},
+		{
+			rules:       "https://radio-t.com   \n https://radio-t.com https://radio-t.com",
+			expectError: true,
+		},
+
+		// valid input, no error
+		{
+			rules: "https://radio-t.com* https://www.radio-t.com*",
+		},
+		{
+			rules: "https://radio-t.com/p/2018/09/22/podcast-616/?with_query=1 https://www.radio-t.com/p/2018/09/22/podcast-616/",
+		},
+		{
+			rules: "https://any.com/p/111 https://any.com/p/222   \n https://any.com/p/333 https://any.com/p/222   \n",
+		},
+	}
+	for _, c := range cases {
+		_, err := NewURLMapper(strings.NewReader(c.rules))
+		if c.expectError {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}

backend/app/migrator/migrator.go

@@ -4,14 +4,14 @@
 package migrator
 
 import (
+	"fmt"
 	"io"
 	"os"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 // Importer defines interface to convert posts from external sources
@@ -24,10 +24,20 @@ type Exporter interface {
 	Export(w io.Writer, siteID string) (int, error)
 }
 
+// Mapper defines interface to convert data in import procedure
+type Mapper interface {
+	URL(url string) string
+}
+
+// MapperMaker defines function that reads rules from reader and
+// returns new Mapper with loaded rules. If rules are not valid
+// it returns error.
+type MapperMaker func(reader io.Reader) (Mapper, error)
+
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
@@ -42,6 +52,8 @@ type ImportParams struct {
 	SiteID    string
 }
 
+var adminUser = store.User{Admin: true}
+
 // ImportComments imports from given provider format and saves to store
 func ImportComments(p ImportParams) (int, error) {
 	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)
@@ -52,18 +64,20 @@ func ImportComments(p ImportParams) (int, error) {
 		importer = &Disqus{DataStore: p.DataStore}
 	case "wordpress":
 		importer = &WordPress{DataStore: p.DataStore}
+	case "commento":
+		importer = &Commento{DataStore: p.DataStore}
 	case "native":
 		importer = &Native{DataStore: p.DataStore}
 	default:
-		return 0, errors.Errorf("unsupported import provider %s", p.Provider)
+		return 0, fmt.Errorf("unsupported import provider %s", p.Provider)
 	}
 
 	fh, err := os.Open(p.InputFile)
 	if err != nil {
-		return 0, errors.Wrapf(err, "can't open import file %s", p.InputFile)
+		return 0, fmt.Errorf("can't open import file %s: %w", p.InputFile, err)
 	}
 
-	defer func() {
+	defer func() { //nolint:gosec // false positive on defer without error check when it's checked here
 		if err = fh.Close(); err != nil {
 			log.Printf("[WARN] can't close %s, %s", p.InputFile, err)
 		}

backend/app/migrator/migrator_test.go

@@ -1,42 +1,38 @@
 package migrator
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestMigrator_ImportDisqus(t *testing.T) {
-	defer func() {
-		os.Remove("/tmp/remark-test.db")
-		os.Remove("/tmp/disqus-test.xml")
-	}()
-
-	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
-	require.Nil(t, err)
+	defer os.Remove("/tmp/remark-test.db")
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.xml",
+		InputFile: "testdata/disqus.xml",
 		SiteID:    "test",
 		Provider:  "disqus",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
 
@@ -46,26 +42,48 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 		os.Remove("/tmp/wordpress-test.xml")
 	}()
 
-	err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
-	require.Nil(t, err)
+	err := os.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0o600)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
 		SiteID:    "test",
 		Provider:  "wordpress",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
+func TestMigrator_ImportCommento(t *testing.T) {
+	defer os.Remove("/tmp/remark-test.db")
+
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
+	size, err := ImportComments(ImportParams{
+		DataStore: dataStore,
+		InputFile: "testdata/commento.json",
+		SiteID:    "test",
+		Provider:  "commento",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(last), "2 comments imported")
+}
+
 func TestMigrator_ImportNative(t *testing.T) {
 	defer func() {
 		os.Remove("/tmp/remark-test.db")
@@ -75,12 +93,13 @@ func TestMigrator_ImportNative(t *testing.T) {
 	data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
-	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
+	err := os.WriteFile("/tmp/disqus-test.r42", []byte(data), 0o600)
+	require.NoError(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -88,19 +107,20 @@ func TestMigrator_ImportNative(t *testing.T) {
 		SiteID:    "radio-t",
 		Provider:  "native",
 	})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	last, err := dataStore.Last("radio-t", 10)
-	assert.Nil(t, err)
+	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
 
 func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	require.NoError(t, err, "create store")
+	dataStore := &service.DataStore{Engine: b}
+	defer dataStore.Close()
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -2,23 +2,28 @@ package migrator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"fmt"
 	"io"
+	"sync/atomic"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
+	"github.com/go-pkgz/syncs"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
+const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
 // {"version": 1, comments:[{...}\n,{}], meta: {meta}}
 // each comments starts from the new line
 type Native struct {
-	DataStore Store
+	DataStore  Store
+	Concurrent int
 }
 
 type meta struct {
@@ -30,9 +35,8 @@ type meta struct {
 // Export all comments to writer as json strings. Each comment is one string, separated by "\n"
 // The final file is a valid json
 func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
-
 	if err = n.exportMeta(siteID, w); err != nil {
-		return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID)
+		return 0, fmt.Errorf("failed to export meta for site %s: %w", siteID, err)
 	}
 
 	topics, err := n.DataStore.List(siteID, 0, 0)
@@ -44,22 +48,21 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	commentsCount := 0
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
-		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
+		if e != nil {
 			return commentsCount, e
 		}
 
 		for _, comment := range comments {
-
 			buf := &bytes.Buffer{}
 			enc := json.NewEncoder(buf)
 			enc.SetEscapeHTML(false)
 
 			if err = enc.Encode(comment); err != nil {
-				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
+				return commentsCount, fmt.Errorf("can't marshal %v: %w", comments, err)
 			}
 			if _, err = w.Write(buf.Bytes()); err != nil {
-				return commentsCount, errors.Wrap(err, "can't write comment data")
+				return commentsCount, fmt.Errorf("can't write comment data: %w", err)
 			}
 			commentsCount++
 		}
@@ -70,41 +73,87 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
-		return errors.Wrap(err, "can't get meta")
+		return fmt.Errorf("can't get meta: %w", err)
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
-		return errors.Wrap(err, "can't encode meta")
+	if err = json.NewEncoder(w).Encode(m); err != nil {
+		return fmt.Errorf("can't encode meta: %w", err)
 	}
 	return nil
 }
 
+// WithMapper wraps reader with url-mapper.
+func WithMapper(reader io.Reader, mapper Mapper) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			log.Printf("[DEBUG] finish write to pipe with %+v", err)
+			if e := w.Close(); e != nil {
+				log.Printf("[WARN] failed close pipe writer with %+v", e)
+			}
+		}()
+
+		// decode from reader and encode to pipe writer
+		dec, enc := json.NewDecoder(reader), json.NewEncoder(w)
+
+		m := meta{}
+		if err = dec.Decode(&m); err != nil {
+			return
+		}
+		for i := range m.Posts {
+			m.Posts[i].URL = mapper.URL(m.Posts[i].URL)
+		}
+		if err = enc.Encode(m); err != nil {
+			return
+		}
+
+		for {
+			comment := store.Comment{}
+			if err = dec.Decode(&comment); err != nil {
+				return
+			}
+			comment.Locator.URL = mapper.URL(comment.Locator.URL)
+			if err = enc.Encode(comment); err != nil {
+				return
+			}
+		}
+	}()
+
+	return r
+}
+
 // Import comments from json strings produced by Remark.Export
 func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
-
 	m := meta{}
 	dec := json.NewDecoder(reader)
 	if err = dec.Decode(&m); err != nil {
-		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
+		return 0, fmt.Errorf("failed to import meta for site %s: %w", siteID, err)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
-		return 0, errors.Errorf("unexpected import file version %d", m.Version)
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+		return 0, fmt.Errorf("unexpected import file version %d", m.Version)
 	}
 
-	if err = n.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := n.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
-	failed := 0
-	total, comments := 0, 0
+	var failed, total, comments int64
+
+	concurrent := defaultConcurrent
+	if n.Concurrent > 0 {
+		concurrent = n.Concurrent
+	}
+	grp := syncs.NewSizedGroup(concurrent, syncs.Preemptive)
 
 	for {
 		comment := store.Comment{}
 		err = dec.Decode(&comment)
+		comment.Imported = true
 		if err == io.EOF {
 			break
 		}
@@ -112,27 +161,33 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		total++
 
 		if err != nil {
+			atomic.AddInt64(&failed, 1)
 			failed++
 			continue
 		}
 
-		if _, err = n.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
+		// write comments in parallel
+		grp.Go(func(context.Context) {
+			if _, e := n.DataStore.Create(comment); e != nil {
+				atomic.AddInt64(&failed, 1)
+				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
+				return
+			}
+			num := atomic.AddInt64(&comments, 1)
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
+			}
+		})
 	}
 
+	grp.Wait()
+
 	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
+		return int(comments), fmt.Errorf("failed to save %d comments", failed)
 	}
 	log.Printf("[INFO] imported %d comments from %d records", comments, total)
 
 	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
 
-	return comments, err
+	return int(comments), err
 }

backend/app/migrator/native_test.go

@@ -4,27 +4,25 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
-
 func TestNative_Export(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // write 2 comments
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
 	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
@@ -32,33 +30,33 @@ func TestNative_Export(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
 	c1 := buf.String()
-	log.Print(c1)
+	t.Log(c1)
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
-	meta := struct {
+	m := struct {
 		Version int                    `json:"version"`
 		Users   []service.UserMetaData `json:"users"`
 		Posts   []service.PostMetaData `json:"posts"`
 	}{}
 
-	require.NoError(t, dec.Decode(&meta), "decode meta")
+	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(meta.Users))
-	assert.Equal(t, "user1", meta.Users[0].ID)
-	assert.Equal(t, false, meta.Users[0].Blocked.Status)
-	assert.Equal(t, true, meta.Users[0].Verified)
-	assert.Equal(t, "user2", meta.Users[1].ID)
-	assert.Equal(t, true, meta.Users[1].Blocked.Status)
-	assert.Equal(t, false, meta.Users[1].Verified)
+	require.Equal(t, 2, len(m.Users))
+	assert.Equal(t, "user1", m.Users[0].ID)
+	assert.Equal(t, false, m.Users[0].Blocked.Status)
+	assert.Equal(t, true, m.Users[0].Verified)
+	assert.Equal(t, "user2", m.Users[1].ID)
+	assert.Equal(t, true, m.Users[1].Blocked.Status)
+	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(meta.Posts))
-	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
-	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+	require.Equal(t, 1, len(m.Posts))
+	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
+	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
 	comments := [3]store.Comment{}
 
@@ -70,28 +68,70 @@ func TestNative_Export(t *testing.T) {
 }
 
 func TestNative_Import(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
-	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00","imported":false}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, size)
 
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
 	assert.Equal(t, "1234", comments[0].ParentID)
 	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.True(t, comments[0].Imported)
 
 	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
 	assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
 	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.True(t, comments[1].Imported)
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
+func TestNative_ImportWithMapper(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
+	// want to remap comments to https://rdt.c
+	rules := `https://radio-t.com* https://rdt.c*`
+	mapper, err := NewURLMapper(strings.NewReader(rules))
+	assert.NoError(t, err)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+	mappedReader := WithMapper(strings.NewReader(inp), mapper)
+
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
+	size, err := r.Import(mappedReader, "radio-t")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+	assert.Equal(t, "https://rdt.c/2", comments[0].Locator.URL)
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+	assert.Equal(t, "https://rdt.c", comments[1].Locator.URL)
 
 	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
 	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
@@ -101,48 +141,51 @@ func TestNative_Import(t *testing.T) {
 }
 
 func TestNative_ImportWrongVersion(t *testing.T) {
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
+
 	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
 	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
 	assert.Equal(t, 0, size)
-
 }
 func TestNative_ImportManyWithError(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // write 2 comments
+	defer teardown()
 
 	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
 
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
-	for i := 0; i < 1200; i++ {
+	for i := 0; i < 100; i++ {
 		buf.WriteString(fmt.Sprintf(goodRec, i))
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")
 
-	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
+	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
+	assert.Equal(t, 100, n)
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
+	assert.NoError(t, err)
+	assert.Equal(t, 100, len(comments))
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
+func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
+	testDB := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
 
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
+	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDB})
+	assert.NoError(t, err)
 
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
@@ -152,7 +195,7 @@ func prep(t *testing.T) *service.DataStore {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
@@ -160,7 +203,10 @@ func prep(t *testing.T) *service.DataStore {
 		User:    store.User{ID: "user2", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	return b
+	return b, func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDB)
+	}
 }

backend/app/migrator/testdata/commento.json

@@ -0,0 +1,69 @@
+{
+  "version": 1,
+  "comments": [
+	{
+	  "commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
+	  "domain": "example.com",
+	  "url": "https://example.com/blog/post/1",
+	  "commenterHex": "anonymous",
+	  "markdown": "Example comment created by user.",
+	  "html": "",
+	  "parentHex": "root",
+	  "score": 1,
+	  "state": "approved",
+	  "creationDate": "2021-03-12T11:21:56Z",
+	  "direction": 0,
+	  "deleted": false
+	},
+	{
+	  "commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
+	  "domain": "example.com",
+	  "url": "https://example.com/blog/post/1",
+	  "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
+	  "markdown": "Example 2 comment created by user.",
+	  "html": "",
+	  "parentHex": "root",
+	  "score": 0,
+	  "state": "approved",
+	  "creationDate": "2021-03-17T12:09:47.722181Z",
+	  "direction": 0,
+	  "deleted": false
+	},
+	{
+	  "commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
+	  "domain": "example.com",
+	  "url": "https://example.com/blog/post/1",
+	  "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
+	  "markdown": "Great reply!",
+	  "html": "",
+	  "parentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
+	  "score": 0,
+	  "state": "approved",
+	  "creationDate": "2021-05-11T15:43:01.852651Z",
+	  "direction": 0,
+	  "deleted": false
+	}
+  ],
+  "commenters": [
+	{
+	  "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
+	  "email": "somegreatmail@gmail.com",
+	  "name": "User5276",
+	  "link": "https://example.com/profile/257",
+	  "photo": "https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
+	  "provider": "sso:example.com",
+	  "joinDate": "2021-03-19T19:27:25.954285Z",
+	  "isModerator": false
+	},
+	{
+	  "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
+	  "email": "moregreatmail@gmail.com",
+	  "name": "Saturnin Uf",
+	  "link": "https://example.com/profile/259",
+	  "photo": "https://secure.gravatar.com/avatar/6481228d190f0286a42bee9041f9b1a1",
+	  "provider": "sso:example.com",
+	  "joinDate": "2021-03-21T12:15:37.536035Z",
+	  "isModerator": false
+	}
+  ]
+}

backend/app/migrator/testdata/disqus-deleted-post.xml

@@ -0,0 +1,150 @@
+`<?xml version="1.0" encoding="utf-8"?>
+<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
+
+    <category dsq:id="707279">
+        <forum>radiot</forum>
+        <title>General</title>
+        <isDefault>true</isDefault>
+    </category>
+
+    <thread dsq:id="247918464">
+        <id/>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T20:46:25Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>98.212.28.115</ipAddress>
+        <isClosed>false</isClosed>
+        <isDeleted>false</isDeleted>
+    </thread>
+
+    <thread dsq:id="247937687">
+        <id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T21:17:17Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>80.250.214.235</ipAddress>
+        <isClosed>true</isClosed>
+        <isDeleted>false</isDeleted>
+    </thread>
+
+
+    <post dsq:id="299619020">
+        <id>3565798471341011339</id>
+        <message>
+            <![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
+        </message>
+        <createdAt>2011-08-31T15:16:29Z</createdAt>
+        <isDeleted>true</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email/>
+            <name>Alexander Blah</name>
+            <isAnonymous>false</isAnonymous>
+            <username>facebook-1787732238</username>
+        </author>
+        <ipAddress>178.178.178.178</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299744309">
+        <id>3029154520436241933</id>
+        <message>
+            <![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
+        </message>
+        <createdAt>2011-08-31T17:44:22Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>mihail.noname@gmail.com</email>
+            <name>mikhail</name>
+            <isAnonymous>false</isAnonymous>
+            <username>mikhail-noname</username>
+        </author>
+        <ipAddress>195.195.195.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299986072">
+        <id>6580890074280459209</id>
+        <message>
+            <![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
+        </message>
+        <createdAt>2011-08-31T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>john.nousername@gmail.com</email>
+            <name>No Username</name>
+            <isAnonymous>false</isAnonymous>
+        </author>
+        <ipAddress>89.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post>
+        <id>12345678890</id>
+        <message>This comment had no ID</message>
+        <createdAt>2011-08-31T22:49:43Z</createdAt>
+        <forum>radiot</forum>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>blah.noname@gmail.com</email>
+            <name>Blah Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>74b9e7568ef6860e93862c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post dsq:id="299986073">
+        <id>6580890074280459219</id>
+        <message>some ugly spam</message>
+        <createdAt>2011-09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>true</isSpam>
+        <author>
+            <email>spam.noname@gmail.com</email>
+            <name>Spam Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="x299986073">
+        <message>some bad comment</message>
+        <createdAt>2011-x09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>123</isSpam>
+        <author>
+            <email>noname@gmail.com</email>
+            <name>Noname</name>
+            <isAnonymous>true</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.39</ipAddress>
+        <thread dsq:id=247937687/>
+    </post>
+
+</disqus>

backend/app/migrator/testdata/disqus-deleted-thread.xml

@@ -0,0 +1,150 @@
+`<?xml version="1.0" encoding="utf-8"?>
+<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
+
+    <category dsq:id="707279">
+        <forum>radiot</forum>
+        <title>General</title>
+        <isDefault>true</isDefault>
+    </category>
+
+    <thread dsq:id="247918464">
+        <id/>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T20:46:25Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>98.212.28.115</ipAddress>
+        <isClosed>false</isClosed>
+        <isDeleted>false</isDeleted>
+    </thread>
+
+    <thread dsq:id="247937687">
+        <id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T21:17:17Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>80.250.214.235</ipAddress>
+        <isClosed>true</isClosed>
+        <isDeleted>true</isDeleted>
+    </thread>
+
+
+    <post dsq:id="299619020">
+        <id>3565798471341011339</id>
+        <message>
+            <![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
+        </message>
+        <createdAt>2011-08-31T15:16:29Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email/>
+            <name>Alexander Blah</name>
+            <isAnonymous>false</isAnonymous>
+            <username>facebook-1787732238</username>
+        </author>
+        <ipAddress>178.178.178.178</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299744309">
+        <id>3029154520436241933</id>
+        <message>
+            <![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
+        </message>
+        <createdAt>2011-08-31T17:44:22Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>mihail.noname@gmail.com</email>
+            <name>mikhail</name>
+            <isAnonymous>false</isAnonymous>
+            <username>mikhail-noname</username>
+        </author>
+        <ipAddress>195.195.195.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299986072">
+        <id>6580890074280459209</id>
+        <message>
+            <![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
+        </message>
+        <createdAt>2011-08-31T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>john.nousername@gmail.com</email>
+            <name>No Username</name>
+            <isAnonymous>false</isAnonymous>
+        </author>
+        <ipAddress>89.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post>
+        <id>12345678890</id>
+        <message>This comment had no ID</message>
+        <createdAt>2011-08-31T22:49:43Z</createdAt>
+        <forum>radiot</forum>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>blah.noname@gmail.com</email>
+            <name>Blah Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>74b9e7568ef6860e93862c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post dsq:id="299986073">
+        <id>6580890074280459219</id>
+        <message>some ugly spam</message>
+        <createdAt>2011-09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>true</isSpam>
+        <author>
+            <email>spam.noname@gmail.com</email>
+            <name>Spam Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="x299986073">
+        <message>some bad comment</message>
+        <createdAt>2011-x09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>123</isSpam>
+        <author>
+            <email>noname@gmail.com</email>
+            <name>Noname</name>
+            <isAnonymous>true</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.39</ipAddress>
+        <thread dsq:id=247937687/>
+    </post>
+
+</disqus>

backend/app/migrator/testdata/disqus.xml

@@ -0,0 +1,150 @@
+`<?xml version="1.0" encoding="utf-8"?>
+<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
+
+    <category dsq:id="707279">
+        <forum>radiot</forum>
+        <title>General</title>
+        <isDefault>true</isDefault>
+    </category>
+
+    <thread dsq:id="247918464">
+        <id/>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T20:46:25Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>98.212.28.115</ipAddress>
+        <isClosed>false</isClosed>
+        <isDeleted>false</isDeleted>
+    </thread>
+
+    <thread dsq:id="247937687">
+        <id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
+        <forum>radiot</forum>
+        <category dsq:id="707279"/>
+        <link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
+        <title>Радио-Т: Радио-Т 229</title>
+        <message/>
+        <createdAt>2011-03-07T21:17:17Z</createdAt>
+        <author>
+            <email>umputun@gmail.com</email>
+            <name>Umputun</name>
+            <isAnonymous>false</isAnonymous>
+            <username>umputun</username>
+        </author>
+        <ipAddress>80.250.214.235</ipAddress>
+        <isClosed>true</isClosed>
+        <isDeleted>false</isDeleted>
+    </thread>
+
+
+    <post dsq:id="299619020">
+        <id>3565798471341011339</id>
+        <message>
+            <![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
+        </message>
+        <createdAt>2011-08-31T15:16:29Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email/>
+            <name>Alexander Blah</name>
+            <isAnonymous>false</isAnonymous>
+            <username>facebook-1787732238</username>
+        </author>
+        <ipAddress>178.178.178.178</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299744309">
+        <id>3029154520436241933</id>
+        <message>
+            <![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
+        </message>
+        <createdAt>2011-08-31T17:44:22Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>mihail.noname@gmail.com</email>
+            <name>mikhail</name>
+            <isAnonymous>false</isAnonymous>
+            <username>mikhail-noname</username>
+        </author>
+        <ipAddress>195.195.195.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="299986072">
+        <id>6580890074280459209</id>
+        <message>
+            <![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
+        </message>
+        <createdAt>2011-08-31T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>john.nousername@gmail.com</email>
+            <name>No Username</name>
+            <isAnonymous>false</isAnonymous>
+        </author>
+        <ipAddress>89.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post>
+        <id>12345678890</id>
+        <message>This comment had no ID</message>
+        <createdAt>2011-08-31T22:49:43Z</createdAt>
+        <forum>radiot</forum>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>blah.noname@gmail.com</email>
+            <name>Blah Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>74b9e7568ef6860e93862c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247918464"/>
+    </post>
+
+    <post dsq:id="299986073">
+        <id>6580890074280459219</id>
+        <message>some ugly spam</message>
+        <createdAt>2011-09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>true</isSpam>
+        <author>
+            <email>spam.noname@gmail.com</email>
+            <name>Spam Noname</name>
+            <isAnonymous>false</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.139</ipAddress>
+        <thread dsq:id="247937687"/>
+    </post>
+
+    <post dsq:id="x299986073">
+        <message>some bad comment</message>
+        <createdAt>2011-x09-30T22:48:43Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>123</isSpam>
+        <author>
+            <email>noname@gmail.com</email>
+            <name>Noname</name>
+            <isAnonymous>true</isAnonymous>
+            <username>google-2c5d77590123</username>
+        </author>
+        <ipAddress>189.89.89.39</ipAddress>
+        <thread dsq:id=247937687/>
+    </post>
+
+</disqus>

backend/app/migrator/wordpress.go

@@ -2,14 +2,14 @@ package migrator
 
 import (
 	"encoding/xml"
+	"fmt"
 	"html"
 	"io"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 const wpTimeLayout = "2006-01-02 15:04:05"
@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -59,9 +60,8 @@ func (w *WordPress) Convert(text string) string {
 
 // Import comments from WP and save to store
 func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
-
-	if err = w.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
+	if e := w.DataStore.DeleteAll(siteID); e != nil {
+		return 0, e
 	}
 
 	commentsCh := w.convert(r, siteID)
@@ -75,9 +75,9 @@ func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
 	}
 
 	if failed > 0 {
-		err = errors.Errorf("failed to save %d comments", failed)
+		err = fmt.Errorf("failed to save %d comments", failed)
 		if passed == 0 {
-			err = errors.New("import failed")
+			err = fmt.Errorf("import failed")
 		}
 	}
 
@@ -87,7 +87,6 @@ func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
 }
 
 func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
-
 	decoder := xml.NewDecoder(r)
 	commentsCh := make(chan store.Comment)
 
@@ -106,12 +105,11 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				break
 			}
 
-			switch el := t.(type) {
-			case xml.StartElement:
+			if el, ok := t.(xml.StartElement); ok {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue
@@ -138,11 +136,12 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								Text:      comment.Content,
 								Timestamp: comment.Date.time,
 								ParentID:  comment.PID,
+								Imported:  true,
 							}
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -6,30 +6,32 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
 
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/service"
 )
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
-	assert.Nil(t, err, "create store")
+	assert.NoError(t, err, "create store")
 
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
+	defer dataStore.Close()
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last(siteID, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
+	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
 
 	c := last[0]
 	assert.Equal(t, "14", c.ID)
@@ -40,16 +42,17 @@ func TestWordPress_Import(t *testing.T) {
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
 	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(posts))
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(posts))
 
 	p := posts[0]
 	assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
 
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
 }
 
@@ -61,7 +64,7 @@ func TestWordPress_Convert(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
+	require.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
 
 	exp1 := store.Comment{
 		ID: "13",
@@ -75,6 +78,7 @@ func TestWordPress_Convert(t *testing.T) {
 			ID:   "wordpress_" + store.EncodeID("Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual"),
 			IP:   "74.200.244.101",
 		},
+		Imported: true,
 	}
 	exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
 	assert.Equal(t, exp1, comments[1])
@@ -88,7 +92,7 @@ func TestWP_Convert_MD(t *testing.T) {
 	for c := range ch {
 		comments = append(comments, c)
 	}
-	assert.Equal(t, 3, len(comments), "3 comments exported")
+	require.Equal(t, 3, len(comments), "3 comments exported")
 
 	assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
 
@@ -188,7 +192,7 @@ var xmlTestWP = `
 		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
 		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
 		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
-		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<category domain="post_tag" nicename="weird-in-a-cant-quite-help-myself-way"><![CDATA[weird in a can't quite help myself way]]></category>
 		<wp:postmeta>
 			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
 			<wp:meta_value><![CDATA[2]]></wp:meta_value>

backend/app/notify/email.go

@@ -0,0 +1,285 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	ntf "github.com/go-pkgz/notify"
+	"github.com/go-pkgz/repeater"
+	"github.com/hashicorp/go-multierror"
+
+	"github.com/umputun/remark42/backend/app/templates"
+)
+
+// EmailParams contain settings for email notifications
+type EmailParams struct {
+	From                     string   // from email address
+	AdminEmails              []string // administrator emails to send copy of comment notification to
+	MsgTemplatePath          string   // path to request message template
+	VerificationSubject      string   // verification message sub
+	VerificationTemplatePath string   // path to verification template
+	SubscribeURL             string   // full subscribe handler URL
+	UnsubscribeURL           string   // full unsubscribe handler URL
+
+	TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
+}
+
+// Email implements notify.Destination for email
+type Email struct {
+	*ntf.Email
+
+	EmailParams
+	msgTmpl    *template.Template // parsed request message template
+	verifyTmpl *template.Template // parsed verification message template
+}
+
+// msgTmplData store data for message from request template execution
+type msgTmplData struct {
+	UserName          string
+	UserPicture       string
+	CommentText       string
+	CommentLink       string
+	CommentDate       time.Time
+	ParentUserName    string
+	ParentUserPicture string
+	ParentCommentText string
+	ParentCommentLink string
+	ParentCommentDate time.Time
+	PostTitle         string
+	Email             string
+	UnsubscribeLink   string
+	ForAdmin          bool
+}
+
+// verifyTmplData store data for verification message template execution
+type verifyTmplData struct {
+	User         string
+	Token        string
+	Email        string
+	Site         string
+	SubscribeURL string
+}
+
+const (
+	defaultVerificationSubject           = "Email verification"
+	defaultEmailTimeout                  = 10 * time.Second
+	defaultEmailTemplatePath             = "email_reply.html.tmpl"
+	defaultEmailVerificationTemplatePath = "email_confirmation_subscription.html.tmpl"
+)
+
+// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
+func NewEmail(emailParams EmailParams, smtpParams ntf.SMTPParams) (*Email, error) {
+	// set up Email emailParams
+	if smtpParams.TimeOut <= 0 {
+		smtpParams.TimeOut = defaultEmailTimeout
+	}
+
+	res := Email{Email: ntf.NewEmail(smtpParams), EmailParams: emailParams}
+
+	if res.VerificationSubject == "" {
+		res.VerificationSubject = defaultVerificationSubject
+	}
+
+	// initialize templates
+	err := res.setTemplates()
+	if err != nil {
+		return nil, fmt.Errorf("can't set templates: %w", err)
+	}
+
+	log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
+		res.Host, res.Username, res.TimeOut)
+
+	return &res, nil
+}
+
+func (e *Email) setTemplates() error {
+	var err error
+	var msgTmplFile, verifyTmplFile []byte
+
+	if e.VerificationTemplatePath == "" {
+		e.VerificationTemplatePath = defaultEmailVerificationTemplatePath
+	}
+
+	if e.MsgTemplatePath == "" {
+		e.MsgTemplatePath = defaultEmailTemplatePath
+	}
+
+	if msgTmplFile, err = templates.Read(e.MsgTemplatePath); err != nil {
+		return fmt.Errorf("can't read message template: %w", err)
+	}
+	if verifyTmplFile, err = templates.Read(e.VerificationTemplatePath); err != nil {
+		return fmt.Errorf("can't read verification template: %w", err)
+	}
+	if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil {
+		return fmt.Errorf("can't parse message template: %w", err)
+	}
+	if e.verifyTmpl, err = template.New("verifyTmpl").Parse(string(verifyTmplFile)); err != nil {
+		return fmt.Errorf("can't parse verification template: %w", err)
+	}
+
+	return nil
+}
+
+// Send email about comment reply to Request.Emails and Email.AdminEmails
+// if they're set.
+// Thread safe
+func (e *Email) Send(ctx context.Context, req Request) error {
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("sending email messages about comment %q aborted due to canceled context", req.Comment.ID)
+	default:
+	}
+
+	result := new(multierror.Error)
+
+	for _, email := range req.Emails {
+		err := e.buildAndSendMessage(ctx, req, email, false)
+		if err != nil {
+			result = multierror.Append(fmt.Errorf("problem sending user email notification to %q: %w", email, err))
+		}
+	}
+
+	for _, email := range e.AdminEmails {
+		err := e.buildAndSendMessage(ctx, req, email, true)
+		if err != nil {
+			result = multierror.Append(fmt.Errorf("problem sending admin email notification to %q: %w", email, err))
+		}
+	}
+
+	return result.ErrorOrNil()
+}
+
+func (e *Email) buildAndSendMessage(ctx context.Context, req Request, email string, forAdmin bool) error {
+	log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
+	msg, err := e.buildMessageFromRequest(req, email, forAdmin)
+	if err != nil {
+		return err
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.Email.Send(
+				ctx,
+				fmt.Sprintf("mailto:%s?from=%s&unsubscribeLink=%s&subject=%s",
+					email,
+					e.From,
+					url.QueryEscape(msg.unsubscribeLink),
+					url.QueryEscape(msg.subject),
+				),
+				msg.body,
+			)
+		})
+}
+
+// SendVerification email verification VerificationRequest.Email if it's set.
+// Thread safe
+func (e *Email) SendVerification(ctx context.Context, req VerificationRequest) error {
+	if req.Email == "" {
+		// this means we can't send this request via Email
+		return nil
+	}
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("sending message to %q aborted due to canceled context", req.User)
+	default:
+	}
+
+	log.Printf("[DEBUG] send verification via %s, user %s", e, req.User)
+	msg, err := e.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
+	if err != nil {
+		return err
+	}
+
+	return repeater.NewDefault(5, time.Millisecond*250).Do(
+		ctx,
+		func() error {
+			return e.Email.Send(
+				ctx,
+				fmt.Sprintf("mailto:%s?from=%s&subject=%s",
+					req.Email,
+					e.From,
+					url.QueryEscape(e.VerificationSubject),
+				),
+				msg,
+			)
+		})
+}
+
+// buildVerificationMessage generates verification email message based on given input
+func (e *Email) buildVerificationMessage(user, email, token, site string) (string, error) {
+	msg := bytes.Buffer{}
+	err := e.verifyTmpl.Execute(&msg, verifyTmplData{
+		User:         user,
+		Token:        token,
+		Email:        email,
+		Site:         site,
+		SubscribeURL: e.SubscribeURL,
+	})
+	if err != nil {
+		return "", fmt.Errorf("error executing template to build verification message: %w", err)
+	}
+	return msg.String(), nil
+}
+
+type commentMessage struct {
+	subject         string
+	body            string
+	unsubscribeLink string
+}
+
+// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
+func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool) (commentMessage, error) {
+	subject := "New reply to your comment"
+	if forAdmin {
+		subject = "New comment to your site"
+	}
+	if req.Comment.PostTitle != "" {
+		subject += fmt.Sprintf(" for %q", req.Comment.PostTitle)
+	}
+
+	token, err := e.TokenGenFn(req.parent.User.ID, email, req.Comment.Locator.SiteID)
+	if err != nil {
+		return commentMessage{}, fmt.Errorf("error creating token for unsubscribe link: %w", err)
+	}
+	unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
+	if forAdmin {
+		unsubscribeLink = ""
+	}
+
+	commentURLPrefix := req.Comment.Locator.URL + uiNav
+	msg := bytes.Buffer{}
+	tmplData := msgTmplData{
+		UserName:        req.Comment.User.Name,
+		UserPicture:     req.Comment.User.Picture,
+		CommentText:     req.Comment.Text,
+		CommentLink:     commentURLPrefix + req.Comment.ID,
+		CommentDate:     req.Comment.Timestamp,
+		PostTitle:       req.Comment.PostTitle,
+		Email:           email,
+		UnsubscribeLink: unsubscribeLink,
+		ForAdmin:        forAdmin,
+	}
+	// in case of message to admin, parent message might be empty
+	if req.Comment.ParentID != "" {
+		tmplData.ParentUserName = req.parent.User.Name
+		tmplData.ParentUserPicture = req.parent.User.Picture
+		tmplData.ParentCommentText = req.parent.Text
+		tmplData.ParentCommentLink = commentURLPrefix + req.parent.ID
+		tmplData.ParentCommentDate = req.parent.Timestamp
+	}
+	err = e.msgTmpl.Execute(&msg, tmplData)
+	if err != nil {
+		return commentMessage{}, fmt.Errorf("error executing template to build comment reply message: %w", err)
+	}
+	return commentMessage{
+		subject:         subject,
+		body:            msg.String(),
+		unsubscribeLink: unsubscribeLink,
+	}, err
+}

backend/app/notify/email_test.go

@@ -0,0 +1,246 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"text/template"
+
+	ntf "github.com/go-pkgz/notify"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+)
+
+func TestEmailNew(t *testing.T) {
+	emailParams := EmailParams{
+		From:                     "test@from",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}
+	smtpParams := ntf.SMTPParams{
+		Host:     "test@host",
+		Port:     1000,
+		TLS:      true,
+		StartTLS: true,
+		Username: "test@username",
+		Password: "test@password",
+	}
+
+	email, err := NewEmail(emailParams, smtpParams)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "email returned")
+
+	assert.NotNil(t, email.msgTmpl, "e.template is set")
+	assert.Equal(t, emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
+	if smtpParams.TimeOut == 0 {
+		assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
+	} else {
+		assert.Equal(t, smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
+	}
+	assert.Equal(t, smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
+	assert.Equal(t, smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
+	assert.Equal(t, smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
+	assert.Equal(t, smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
+	assert.Equal(t, smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
+	assert.Equal(t, smtpParams.StartTLS, email.StartTLS, "emailParams.TLS unchanged after creation")
+	assert.Equal(t, "email: with username 'test@username' at server test@host:1000 with TLS with StartTLS", email.String())
+}
+
+func Test_initTemplatesErr(t *testing.T) {
+	testSet := []struct {
+		name        string
+		errText     string
+		emailParams EmailParams
+	}{
+		{
+			name:    "with wrong path to verification template",
+			errText: "notfound.tmpl: file does not exist",
+			emailParams: EmailParams{
+				VerificationTemplatePath: "notfound.tmpl",
+			},
+		},
+		{
+			name:    "with wrong path to message template",
+			errText: "notfound.tmpl: file does not exist",
+			emailParams: EmailParams{
+				MsgTemplatePath: "notfound.tmpl",
+			},
+		},
+		{
+			name:    "with error on read verification template",
+			errText: "can't parse verification template: template: verifyTmpl",
+			emailParams: EmailParams{
+				VerificationTemplatePath: "testdata/bad.html.tmpl",
+				MsgTemplatePath:          "testdata/msg.html.tmpl",
+			},
+		},
+		{
+			name:    "with error on read message template",
+			errText: "can't parse message template: template: msgTmpl",
+			emailParams: EmailParams{
+				VerificationTemplatePath: "testdata/verification.html.tmpl",
+				MsgTemplatePath:          "testdata/bad.html.tmpl",
+			},
+		},
+	}
+
+	for _, d := range testSet {
+		d := d
+		t.Run(d.name, func(t *testing.T) {
+			e, err := NewEmail(d.emailParams, ntf.SMTPParams{})
+			require.Error(t, err)
+			require.Nil(t, e)
+			assert.Contains(t, err.Error(), d.errText)
+		})
+	}
+}
+
+func TestEmailSendErrors(t *testing.T) {
+	var err error
+	e := Email{}
+	e.TokenGenFn = TokenGenFn
+
+	e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.SendVerification(context.Background(), VerificationRequest{Email: "bad@example.org", Token: "some"}),
+		"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
+
+	e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
+	assert.NoError(t, err)
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
+		"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\": "+
+			"error executing template to build comment reply message: "+
+			"template: test:1:2: executing \"test\" at <.Test>: "+
+			"can't evaluate field Test in type notify.msgTmplData\n\n")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
+		"sending email messages about comment \"999\" aborted due to canceled context")
+
+	assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad@example.org"}}),
+		"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\":"+
+			" error creating token for unsubscribe link: token generation error\n\n")
+}
+
+func TestEmailSend_ExitConditions(t *testing.T) {
+	email, err := NewEmail(EmailParams{
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, ntf.SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email, "expecting email returned")
+	// prevent triggering e.autoFlush creation
+	emptyRequest := Request{Comment: store.Comment{ID: "999"}}
+	assert.NoError(t, email.Send(context.Background(), emptyRequest),
+		"Message without Emails and AdminEmails is not sent and returns nil")
+}
+
+func TestEmail_Send(t *testing.T) {
+	email, err := NewEmail(EmailParams{
+		From:                     "from@example.org",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, ntf.SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	email.TokenGenFn = TokenGenFn
+	email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
+	req := Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
+		parent:  store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
+		Emails:  []string{"test@example.org"},
+	}
+	assert.Contains(t, email.Send(context.Background(), req).Error(), "problem sending user email notification to \"test@example.org\"")
+	// test buildMessageFromRequest separately for message text
+	msg, err := email.buildMessageFromRequest(req, req.Emails[0], false)
+	assert.NoError(t, err)
+	assert.Equal(t, `
+	New reply from test_user on your comment to «test_title»
+
+User: test_user
+01.01.0001 at 00:00
+Comment: 
+test@example.org  for parent_user
+Unsubscribe link: https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token
+`, msg.body)
+	assert.Equal(t, "https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token", msg.unsubscribeLink)
+	assert.Equal(t, `New reply to your comment for "test_title"`, msg.subject)
+
+	// send email to both user and admin, without parent set
+	email.AdminEmails = []string{"admin@example.org"}
+	req = Request{
+		Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
+		Emails:  []string{"test@example.org"},
+	}
+	assert.Error(t, email.Send(context.Background(), req))
+	msg, err = email.buildMessageFromRequest(req, email.AdminEmails[0], true)
+	assert.NoError(t, err)
+	assert.Equal(t, `
+New comment from test_user on your site  to «test_title»
+
+User: test_user
+01.01.0001 at 00:00
+Comment: 
+admin@example.org 
+`, msg.body)
+	assert.Equal(t, `New comment to your site for "test_title"`, msg.subject)
+	assert.Empty(t, msg.unsubscribeLink)
+}
+
+func TestEmail_SendVerification(t *testing.T) {
+	email, err := NewEmail(EmailParams{
+		From:                     "from@example.org",
+		VerificationTemplatePath: "testdata/verification.html.tmpl",
+		MsgTemplatePath:          "testdata/msg.html.tmpl",
+	}, ntf.SMTPParams{})
+	assert.NoError(t, err)
+	assert.NotNil(t, email)
+	email.TokenGenFn = TokenGenFn
+	// proper VerificationRequest without email
+	req := VerificationRequest{
+		SiteID: "remark",
+		User:   "test_username",
+		Token:  "secret_",
+	}
+	assert.NoError(t, email.SendVerification(context.Background(), req))
+
+	// proper VerificationRequest with email
+	req.Email = "test@example.org"
+	assert.Error(t, email.SendVerification(context.Background(), req), "failed to make smtp client")
+
+	// VerificationRequest with canceled context
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t, email.SendVerification(ctx, req), "sending message to \"test_username\" aborted due to canceled context")
+
+	// test buildVerificationMessage separately for message text
+	res, err := email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
+	assert.NoError(t, err)
+	assert.Equal(t, res, `Confirmation for test_username on site remark
+Token:secret_
+Sent to test@example.org
+
+`)
+	assert.Contains(t, res, `secret_`)
+	assert.NotContains(t, res, `https://example.org/`)
+	email.SubscribeURL = "https://example.org/subscribe.html?token="
+	res, err = email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
+	assert.NoError(t, err)
+	assert.Equal(t, res, `Confirmation for test_username on site remark
+Subscribe url: https://example.org/subscribe.html?token=secret_
+Token:secret_
+Sent to test@example.org
+
+`)
+}
+
+func TokenGenFn(user, _, _ string) (string, error) {
+	if user == "error" {
+		return "", fmt.Errorf("token generation error")
+	}
+	return "token", nil
+}

backend/app/notify/notify.go

@@ -9,14 +9,15 @@ import (
 
 	log "github.com/go-pkgz/lgr"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 // Service delivers notifications to multiple destinations
 type Service struct {
-	dataService  Store
-	destinations []Destination
-	queue        chan request
+	dataService       Store
+	destinations      []Destination
+	queue             chan Request
+	verificationQueue chan VerificationRequest
 
 	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
 	ctx    context.Context
@@ -26,16 +27,34 @@ type Service struct {
 // Destination defines interface for a given destination service, like telegram, email and so on
 type Destination interface {
 	fmt.Stringer
-	Send(ctx context.Context, req request) error
+	Send(context.Context, Request) error
+	SendVerification(context.Context, VerificationRequest) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
-	Get(locator store.Locator, id string) (store.Comment, error)
+	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
+	GetUserEmail(siteID string, userID string) (string, error)
+	GetUserTelegram(siteID string, userID string) (string, error)
 }
-type request struct {
-	comment store.Comment
-	parent  store.Comment
+
+// used for email and telegram retrieval from user details
+type getUserDetail func(string, string) (string, error)
+
+// Request notification for a Comment
+type Request struct {
+	Comment   store.Comment
+	parent    store.Comment
+	Emails    []string
+	Telegrams []string
+}
+
+// VerificationRequest notification for user
+type VerificationRequest struct {
+	SiteID string
+	User   string
+	Email  string // if set, send email only
+	Token  string
 }
 
 const defaultQueueSize = 100
@@ -48,11 +67,12 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	res := Service{
-		dataService:  dataService,
-		queue:        make(chan request, size),
-		destinations: destinations,
-		ctx:          ctx,
-		cancel:       cancel,
+		dataService:       dataService,
+		queue:             make(chan Request, size),
+		verificationQueue: make(chan VerificationRequest, size),
+		destinations:      destinations,
+		ctx:               ctx,
+		cancel:            cancel,
 	}
 	if len(destinations) > 0 {
 		go res.do()
@@ -61,21 +81,62 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
 	return &res
 }
 
-// Submit comment to internal channel if not busy, drop if can't send
-func (s *Service) Submit(comment store.Comment) {
+// Submit Request to internal channel if not busy, drop if can't send
+func (s *Service) Submit(req Request) {
 	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
 		return
 	}
-	parentComment := store.Comment{}
-	if s.dataService != nil {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
-			parentComment = p
+	if s.dataService != nil && req.Comment.ParentID != "" {
+		if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
+			req.parent = p
+			req.Emails = s.getNotificationTargets(req, p, s.dataService.GetUserEmail)
+			req.Telegrams = s.getNotificationTargets(req, p, s.dataService.GetUserTelegram)
 		}
 	}
 	select {
-	case s.queue <- request{comment: comment, parent: parentComment}:
+	case s.queue <- req:
 	default:
-		log.Printf("[WARN] can't send comment notification to queue, %+v", comment)
+		log.Printf("[WARN] can't send notification to queue, %+v", req.Comment)
+	}
+}
+
+// getNotificationTargets returns list of notification targets (like email or telegram username) for users
+// interested in notifications for provided comment.
+// Targets are not added to the returned list in case the original message
+// is from the same user as the notification receiver.
+// Results are deduplicated.
+func (s *Service) getNotificationTargets(
+	req Request,
+	notifyComment store.Comment,
+	getUserDetail getUserDetail,
+) (result []string) {
+	// add current user email only if the user is not the one who wrote the original comment
+	if notifyComment.User.ID != req.Comment.User.ID {
+		detail, err := getUserDetail(req.Comment.Locator.SiteID, notifyComment.User.ID)
+		if err != nil {
+			log.Printf("[WARN] can't read notification detail for %s, %v", notifyComment.User.ID, err)
+		}
+		if detail != "" {
+			result = append(result, detail)
+		}
+	}
+	if notifyComment.ParentID != "" {
+		if p, err := s.dataService.Get(req.Comment.Locator, notifyComment.ParentID, store.User{}); err == nil {
+			result = append(result, s.getNotificationTargets(req, p, getUserDetail)...)
+		}
+	}
+	return deduplicateStrings(result)
+}
+
+// SubmitVerification to internal channel if not busy, drop if can't send
+func (s *Service) SubmitVerification(req VerificationRequest) {
+	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
+		return
+	}
+	select {
+	case s.verificationQueue <- req:
+	default:
+		log.Printf("[WARN] can't send verification to queue, %s for %s", req.User, req.Email)
 	}
 }
 
@@ -84,6 +145,7 @@ func (s *Service) Close() {
 	if s.queue != nil {
 		log.Print("[DEBUG] close notifier")
 		close(s.queue)
+		close(s.verificationQueue)
 		s.cancel()
 		<-s.ctx.Done()
 	}
@@ -91,21 +153,60 @@ func (s *Service) Close() {
 }
 
 func (s *Service) do() {
-	for c := range s.queue {
-		var wg sync.WaitGroup
-		wg.Add(len(s.destinations))
-		for _, dest := range s.destinations {
-			go func(d Destination) {
-				if err := d.Send(s.ctx, c); err != nil {
-					log.Printf("[WARN] failed to send to %s, %s", d, err)
-				}
-				wg.Done()
-			}(dest)
+	defer log.Print("[WARN] terminated notifier")
+	var wg sync.WaitGroup
+	for {
+		select {
+		case c, ok := <-s.queue:
+			if !ok {
+				return
+			}
+			wg.Add(len(s.destinations))
+			for _, dest := range s.destinations {
+				go func(d Destination) {
+					if err := d.Send(s.ctx, c); err != nil {
+						log.Printf("[WARN] failed to send to %s, %s", d, err)
+					}
+					wg.Done()
+				}(dest)
+			}
+			wg.Wait()
+		case v, ok := <-s.verificationQueue:
+			if !ok {
+				return
+			}
+			wg.Add(len(s.destinations))
+			for _, dest := range s.destinations {
+				go func(d Destination) {
+					if err := d.SendVerification(s.ctx, v); err != nil {
+						log.Printf("[WARN] failed to send to %s, %s", d, err)
+					}
+					wg.Done()
+				}(dest)
+			}
+			wg.Wait()
+		case <-s.ctx.Done():
+			return
 		}
-		wg.Wait()
 	}
-	log.Print("[WARN] terminated notifier")
 }
 
 // NopService is do-nothing notifier, without destinations
 var NopService = &Service{}
+
+// deduplicateStrings returns provided slice of strings will all duplicates removed.
+// Resulting slice is not sorted.
+func deduplicateStrings(source []string) []string {
+	set := make(map[string]struct{})
+
+	for _, k := range source {
+		set[k] = struct{}{}
+	}
+
+	result := make([]string, 0, len(set))
+	for k := range set {
+		result = append(result, k)
+	}
+
+	return result
+}

backend/app/notify/notify_mock.go

@@ -0,0 +1,69 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// MockDest is a destination mock
+type MockDest struct {
+	data             []Request
+	verificationData []VerificationRequest
+	id               int
+	closed           bool
+	lock             sync.Mutex
+}
+
+// Send mock
+func (m *MockDest) Send(ctx context.Context, r Request) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.data = append(m.data, r)
+		log.Printf("sent %s -> %d", r.Comment.ID, m.id)
+	case <-ctx.Done():
+		log.Printf("ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+// SendVerification mock
+func (m *MockDest) SendVerification(ctx context.Context, v VerificationRequest) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(10 * time.Millisecond):
+		m.verificationData = append(m.verificationData, v)
+		log.Printf("sent verification %s -> %d", v.User, m.id)
+	case <-ctx.Done():
+		log.Printf("verification ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+// Get mock
+func (m *MockDest) Get() []Request {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]Request, len(m.data))
+	copy(res, m.data)
+	return res
+}
+
+// GetVerify mock
+func (m *MockDest) GetVerify() []VerificationRequest {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]VerificationRequest, len(m.verificationData))
+	copy(res, m.verificationData)
+	return res
+}
+
+func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }

backend/app/notify/notify_test.go

@@ -1,90 +1,119 @@
 package notify
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"math/rand"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
 func TestService_NoDestinations(t *testing.T) {
-	s := NewService(nil, 1)
+	s := NewService(nil, 0)
+	assert.Equal(t, defaultQueueSize, cap(s.queue))
 	assert.NotNil(t, s)
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
-	s.Submit(store.Comment{ID: "123"})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
+	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "101"})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	assert.Equal(t, 3, len(d1.get()), "got all comments to d1")
-	assert.Equal(t, 3, len(d2.get()), "got all comments to d2")
+	require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
+	require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
 
-	assert.Equal(t, "100", d1.get()[0].comment.ID)
-	assert.Equal(t, "101", d1.get()[1].comment.ID)
-	assert.Equal(t, "102", d1.get()[2].comment.ID)
+	assert.Equal(t, "100", d1.Get()[0].Comment.ID)
+	assert.Equal(t, "101", d1.Get()[1].Comment.ID)
+	assert.Equal(t, "102", d1.Get()[2].Comment.ID)
 }
 
 func TestService_WithDrops(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 1, d1, d2)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "100"})
-	s.Submit(store.Comment{ID: "101"})
-	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "102"})
-	time.Sleep(time.Millisecond * 110)
+	s.Submit(Request{Comment: store.Comment{ID: "100"}})
+	s.Submit(Request{Comment: store.Comment{ID: "101"}})
+	s.Submit(Request{Comment: store.Comment{ID: "102"}})
+	time.Sleep(time.Millisecond * 21)
 	s.Close()
 
-	s.Submit(store.Comment{ID: "111"}) // safe to send after close
+	s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
 
-	assert.Equal(t, 2, len(d1.get()), "one comment dropped from d1")
-	assert.Equal(t, 2, len(d2.get()), "one comment dropped from d2")
+	assert.LessOrEqual(t, len(d1.Get()), 2, "at least one comment from three dropped from d1, got: %v", d1.Get())
+	assert.LessOrEqual(t, len(d2.Get()), 2, "at least one comment from three dropped from d2, got: %v", d2.Get())
+}
+
+func TestService_SubmitVerificationWithDrops(t *testing.T) {
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
+	s := NewService(nil, 1, d1, d2)
+	assert.NotNil(t, s)
+
+	s.SubmitVerification(VerificationRequest{
+		SiteID: "remark",
+		User:   "testUser",
+		Email:  "test@example.org",
+		Token:  "testToken",
+	})
+	s.SubmitVerification(VerificationRequest{})
+	s.SubmitVerification(VerificationRequest{})
+	time.Sleep(time.Millisecond * 21)
+	s.Close()
+
+	s.SubmitVerification(VerificationRequest{}) // safe to send after close
+
+	assert.LessOrEqual(t, len(d2.GetVerify()), 2, "one request from three dropped from d2, got: %v", d2.GetVerify())
+
+	verifyDest := d1.GetVerify()
+	require.LessOrEqual(t, len(verifyDest), 2, "one request from three dropped from d1, got: %v", verifyDest)
+	assert.Equal(t, "remark", verifyDest[0].SiteID)
+	assert.Equal(t, "testUser", verifyDest[0].User)
+	assert.Equal(t, "test@example.org", verifyDest[0].Email)
+	assert.Equal(t, "testToken", verifyDest[0].Token)
 }
 
 func TestService_Many(t *testing.T) {
-	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
 	s := NewService(nil, 5, d1, d2)
 	assert.NotNil(t, s)
 
 	for i := 0; i < 10; i++ {
-		s.Submit(store.Comment{ID: fmt.Sprintf("%d", 100+i)})
-		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(200)))
+		s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
+		s.SubmitVerification(VerificationRequest{User: fmt.Sprintf("%d", 100+i)})
+		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
 	}
 	s.Close()
 	time.Sleep(time.Millisecond * 10)
 
-	assert.NotEqual(t, 10, len(d1.get()), "some comments dropped from d1")
-	assert.NotEqual(t, 10, len(d2.get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d1.GetVerify()), "some verifications dropped from d1")
+	assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
+	assert.NotEqual(t, 10, len(d2.GetVerify()), "some verifications dropped from d2")
 
 	assert.True(t, d1.closed)
 	assert.True(t, d2.closed)
+	assert.Equal(t, "mock id=1, closed=true", d1.String())
 }
 
 func TestService_WithParent(t *testing.T) {
-	dest := &mockDest{id: 1}
+	dest := &MockDest{id: 1}
 	dataStore := &mockStore{data: map[string]store.Comment{}}
 
 	dataStore.data["p1"] = store.Comment{ID: "p1"}
@@ -93,63 +122,184 @@ func TestService_WithParent(t *testing.T) {
 	s := NewService(dataStore, 1, dest)
 	assert.NotNil(t, s)
 
-	s.Submit(store.Comment{ID: "c1", ParentID: "p1"})
+	s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
 	time.Sleep(time.Millisecond * 110)
-	s.Submit(store.Comment{ID: "c11", ParentID: "p11"})
+	s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
 	time.Sleep(time.Millisecond * 110)
 	s.Close()
 
-	destRes := dest.get()
-	assert.Equal(t, 2, len(destRes), "two comment notified")
-	assert.Equal(t, "p1", destRes[0].comment.ParentID)
+	destRes := dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ParentID)
 	assert.Equal(t, "p1", destRes[0].parent.ID)
-	assert.Equal(t, "p11", destRes[1].comment.ParentID)
+	assert.Equal(t, "p11", destRes[1].Comment.ParentID)
 	assert.Equal(t, "", destRes[1].parent.ID)
 }
 
+func TestService_EmailRetrieval(t *testing.T) {
+	dest := &MockDest{id: 1}
+	dataStore := &mockStore{data: map[string]store.Comment{}, userDetails: map[string]string{}}
+
+	dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p1", User: store.User{ID: "u2"}}
+	dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
+	dataStore.userDetails["u1"] = "u1@example.com"
+
+	s := NewService(dataStore, 1, dest)
+	assert.NotNil(t, s)
+
+	// one comment, one notification
+	s.Submit(Request{Comment: dataStore.data["p1"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes := dest.Get()
+	require.Equal(t, 1, len(destRes), "one comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ID)
+	assert.Empty(t, destRes[0].parent)
+	assert.Empty(t, destRes[0].Emails)
+
+	// reply to the first comment, same comment as one in original comment
+	s.Submit(Request{Comment: dataStore.data["p2"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p2", destRes[1].Comment.ID)
+	assert.Equal(t, "p1", destRes[1].parent.ID)
+	assert.Equal(t, "u1", destRes[1].parent.User.ID)
+	assert.Empty(t, destRes[1].Emails, "u1 is not notified they are the one who left the comment")
+
+	// another reply to the first comment, another user
+	s.Submit(Request{Comment: dataStore.data["p3"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 3, len(destRes), "three comment notified")
+	assert.Equal(t, "p3", destRes[2].Comment.ID)
+	assert.Equal(t, "p1", destRes[2].parent.ID)
+	assert.Equal(t, "u1", destRes[2].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
+
+	// reply to the last comment by another user, should trigger email retrieval error
+	s.Submit(Request{Comment: dataStore.data["p4"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 4, len(destRes), "four comment notified")
+	assert.Equal(t, "p4", destRes[3].Comment.ID)
+	assert.Equal(t, "p3", destRes[3].parent.ID)
+	assert.Equal(t, "u2", destRes[3].parent.User.ID)
+	assert.Empty(t, destRes[3].Emails, "no email can be retrieved for u2")
+
+	s.Close()
+}
+
+func TestService_Recursive(t *testing.T) {
+	dest := &MockDest{id: 1}
+	dataStore := &mockStore{data: map[string]store.Comment{}, userDetails: map[string]string{}}
+
+	dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
+	dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u2"}}
+	dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p2", User: store.User{ID: "u3"}}
+	dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
+	dataStore.data["p5"] = store.Comment{ID: "p5", ParentID: "p4", User: store.User{ID: "u4"}}
+	dataStore.userDetails["u1"] = "u1@example.com"
+	// second comment goes without email address for notification
+	dataStore.userDetails["u3"] = "u3@example.com"
+
+	s := NewService(dataStore, 1, dest)
+	assert.NotNil(t, s)
+
+	// one comment from u1 with email set
+	s.Submit(Request{Comment: dataStore.data["p1"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes := dest.Get()
+	require.Equal(t, 1, len(destRes), "one comment notified")
+	assert.Equal(t, "p1", destRes[0].Comment.ID)
+	assert.Empty(t, destRes[0].parent)
+	assert.Empty(t, destRes[0].Emails)
+
+	// reply to the first comment from u2 without email set
+	s.Submit(Request{Comment: dataStore.data["p2"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p2", destRes[1].Comment.ID)
+	assert.Equal(t, "p1", destRes[1].parent.ID)
+	assert.Equal(t, "u1", destRes[1].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[1].Emails)
+
+	// reply to the second comment from u3 with email set
+	s.Submit(Request{Comment: dataStore.data["p3"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 3, len(destRes), "three comment notified")
+	assert.Equal(t, "p3", destRes[2].Comment.ID)
+	assert.Equal(t, "p2", destRes[2].parent.ID)
+	assert.Equal(t, "u2", destRes[2].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
+
+	// reply to the third comment from u1 (author of the first comment), only u3 should be notified
+	s.Submit(Request{Comment: dataStore.data["p4"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 4, len(destRes), "four comment notified once each")
+	assert.Equal(t, "p4", destRes[3].Comment.ID)
+	assert.Equal(t, "p3", destRes[3].parent.ID)
+	assert.Equal(t, "u3", destRes[3].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u3@example.com"}, destRes[3].Emails, "u1 is not notified they are the one who left the comment")
+
+	// reply to the fourth comment from u4, u1 and u3 should be notified once as a result
+	s.Submit(Request{Comment: dataStore.data["p5"]})
+	time.Sleep(time.Millisecond * 110)
+
+	destRes = dest.Get()
+	require.Equal(t, 5, len(destRes), "four comment notified once each")
+	assert.Equal(t, "p5", destRes[4].Comment.ID)
+	assert.Equal(t, "p4", destRes[4].parent.ID)
+	assert.Equal(t, "u1", destRes[4].parent.User.ID)
+	assert.ElementsMatch(t, []string{"u1@example.com", "u3@example.com"}, destRes[4].Emails, "u3 and u1 notified once")
+
+	s.Close()
+}
+
 func TestService_Nop(t *testing.T) {
 	s := NopService
-	s.Submit(store.Comment{})
+	s.Submit(Request{Comment: store.Comment{}})
 	s.Close()
 	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
 }
 
-type mockDest struct {
-	data   []request
-	id     int
-	closed bool
-	lock   sync.Mutex
+type mockStore struct {
+	data        map[string]store.Comment
+	userDetails map[string]string
 }
 
-func (m *mockDest) Send(ctx context.Context, r request) error {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	select {
-	case <-time.After(100 * time.Millisecond):
-		m.data = append(m.data, r)
-		log.Printf("sent %s -> %d", r.comment.ID, m.id)
-	case <-ctx.Done():
-		log.Printf("ctx closed %d", m.id)
-		m.closed = true
+func (m mockStore) getUserDetail(userID string) (string, error) {
+	detail, ok := m.userDetails[userID]
+	if !ok {
+		return "", fmt.Errorf("no such user")
 	}
-	return nil
+	return detail, nil
 }
 
-func (m *mockDest) get() []request {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-	res := make([]request, len(m.data))
-	copy(res, m.data)
-	return res
-}
-func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
-
-type mockStore struct{ data map[string]store.Comment }
-
-func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
-		return store.Comment{}, errors.New("no such id")
+		return store.Comment{}, fmt.Errorf("no such id")
 	}
 	return res, nil
 }
+
+func (m mockStore) GetUserEmail(_, userID string) (string, error) {
+	return m.getUserDetail(userID)
+}
+
+func (m mockStore) GetUserTelegram(_, userID string) (string, error) {
+	return m.getUserDetail(userID)
+}

backend/app/notify/slack.go

@@ -0,0 +1,61 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	log "github.com/go-pkgz/lgr"
+	ntf "github.com/go-pkgz/notify"
+)
+
+// Slack implements notify.Destination for Slack
+type Slack struct {
+	*ntf.Slack
+
+	channelName string
+}
+
+// NewSlack makes Slack bot for notifications
+func NewSlack(token, channelName string) *Slack {
+	log.Printf("[DEBUG] create new slack notifier for chan %s", channelName)
+	if channelName == "" {
+		channelName = "general"
+	}
+
+	return &Slack{Slack: ntf.NewSlack(token), channelName: channelName}
+}
+
+// Send to Slack channel
+func (s *Slack) Send(ctx context.Context, req Request) error {
+	log.Printf("[DEBUG] send slack notification, comment id %s", req.Comment.ID)
+
+	user := req.Comment.User.Name
+	if req.Comment.ParentID != "" {
+		user += " → " + req.parent.User.Name
+	}
+
+	title := "↦ original comment"
+	if req.Comment.PostTitle != "" {
+		title = "↦ " + req.Comment.PostTitle
+	}
+
+	destination := fmt.Sprintf(
+		"slack:%s?title=%s&attachmentText=%s&titleLink=%s",
+		s.channelName,
+		url.QueryEscape(title),
+		url.QueryEscape(req.Comment.Orig),
+		url.QueryEscape(req.Comment.Locator.URL+uiNav+req.Comment.ID),
+	)
+
+	return s.Slack.Send(ctx, destination, "New comment from "+user)
+}
+
+// SendVerification is not implemented for Slack
+func (s *Slack) SendVerification(_ context.Context, _ VerificationRequest) error {
+	return nil
+}
+
+func (s *Slack) String() string {
+	return s.Slack.String() + " for channel " + s.channelName + ""
+}

backend/app/notify/slack_test.go

@@ -0,0 +1,40 @@
+package notify
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store"
+)
+
+func TestSlack_New(t *testing.T) {
+	ts := NewSlack("", "")
+	assert.NotNil(t, ts)
+	assert.Equal(t, "general", ts.channelName)
+}
+
+func TestSlack_Send(t *testing.T) {
+	ts := NewSlack("", "")
+
+	c := store.Comment{PostTitle: "test title", Text: "some text", ParentID: "1", ID: "999"}
+	c.User.Name = "from"
+	cp := store.Comment{Text: "some parent text"}
+	cp.User.Name = "to"
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	err := ts.Send(ctx, Request{Comment: c, parent: cp})
+	assert.Error(t, err)
+}
+
+func TestSlack_Name(t *testing.T) {
+	tb := NewSlack("", "test-channel")
+	assert.Equal(t, "slack notifications destination for channel test-channel", tb.String())
+}
+
+func TestSlack_SendVerification(t *testing.T) {
+	ts := NewSlack("", "")
+	assert.NoError(t, ts.SendVerification(context.Background(), VerificationRequest{}))
+}

backend/app/notify/telegram.go

@@ -1,145 +1,115 @@
 package notify
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"html"
-	"net/http"
-	"strconv"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/repeater"
-	"github.com/pkg/errors"
+	ntf "github.com/go-pkgz/notify"
+	"github.com/hashicorp/go-multierror"
 )
 
+// TelegramParams contain settings for telegram notifications
+type TelegramParams struct {
+	AdminChannelID       string        // unique identifier for the target chat or username of the target channel (in the format @channelusername)
+	Token                string        // token for telegram bot API interactions
+	Timeout              time.Duration // http client timeout
+	UserNotifications    bool          // flag which enables user notifications
+	ErrorMsg, SuccessMsg string        // messages for successful and unsuccessful subscription requests to bot
+}
+
 // Telegram implements notify.Destination for telegram
 type Telegram struct {
-	channelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
-	token     string
-	apiPrefix string
-	timeout   time.Duration
-}
+	*ntf.Telegram
 
-const telegramTimeOut = 5000 * time.Millisecond
-const telegramAPIPrefix = "https://api.telegram.org/bot"
+	AdminChannelID    string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
+	UserNotifications bool   // flag which enables user notifications
+}
 
 // NewTelegram makes telegram bot for notifications
-func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
-
-	if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
-		channelID = "@" + channelID // if channelID not a number enforce @ prefix
-	}
-
-	res := Telegram{channelID: channelID, token: token, apiPrefix: api, timeout: timeout}
-	if res.apiPrefix == "" {
-		res.apiPrefix = telegramAPIPrefix
-	}
-	if res.timeout == 0 {
-		res.timeout = telegramTimeOut
-	}
-	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
-
-	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
-		client := http.Client{Timeout: telegramTimeOut}
-		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
-		if err != nil {
-			return errors.Wrap(err, "can't initialize telegram notifications")
-		}
-		defer func() {
-			if err = resp.Body.Close(); err != nil {
-				log.Printf("[WARN] can't close request body, %s", err)
-			}
-		}()
-
-		if resp.StatusCode != http.StatusOK {
-			return errors.Errorf("unexpected telegram status code %d", resp.StatusCode)
-		}
-
-		tgResp := struct {
-			OK     bool `json:"ok"`
-			Result struct {
-				FirstName string `json:"first_name"`
-				ID        uint64 `json:"id"`
-				IsBot     bool   `json:"is_bot"`
-				UserName  string `json:"username"`
-			}
-		}{}
-
-		if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
-			return errors.Wrap(err, "can't decode response")
-		}
-
-		if !tgResp.OK || !tgResp.Result.IsBot {
-			return errors.Errorf("unexpected telegram response %+v", tgResp)
-		}
-		return nil
+func NewTelegram(params TelegramParams) (*Telegram, error) {
+	client, err := ntf.NewTelegram(ntf.TelegramParams{
+		Token:      params.Token,
+		Timeout:    params.Timeout,
+		ErrorMsg:   params.ErrorMsg,
+		SuccessMsg: params.SuccessMsg,
 	})
+	if err != nil {
+		return nil, err
+	}
 
-	return &res, err
+	return &Telegram{Telegram: client, AdminChannelID: params.AdminChannelID, UserNotifications: params.UserNotifications}, nil
 }
 
-// Send to telegram channel
-func (t *Telegram) Send(ctx context.Context, req request) error {
-	client := http.Client{Timeout: telegramTimeOut}
-	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
+// Send to telegram recipients
+func (t *Telegram) Send(ctx context.Context, req Request) error {
+	log.Printf("[DEBUG] send telegram notification for comment ID %s", req.Comment.ID)
+	result := new(multierror.Error)
 
-	from := req.comment.User.Name
-	if req.comment.ParentID != "" {
-		from += " → " + req.parent.User.Name
-	}
-	from = "*" + from + "*"
-	link := fmt.Sprintf("↦ [original comment](%s)", req.comment.Locator.URL+uiNav+req.comment.ID)
-	if req.comment.PostTitle != "" {
-		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
-	}
-	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
-		t.apiPrefix, t.token, t.channelID)
+	msg := t.buildMessage(req)
 
-	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
-	msg = html.UnescapeString(msg)
-	body := struct {
-		Text string `json:"text"`
-	}{Text: msg}
-
-	b, err := json.Marshal(body)
-	if err != nil {
-		return errors.Wrap(err, "failed to make telegram body")
-	}
-
-	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
-	if err != nil {
-		return errors.Wrap(err, "failed to make telegram request")
-	}
-
-	r = r.WithContext(ctx)
-	resp, err := client.Do(r)
-	if err != nil {
-		return errors.Wrap(err, "failed to get telegram response")
-	}
-	defer func() {
-		if err = resp.Body.Close(); err != nil {
-			log.Printf("[WARN] can't close request body, %s", err)
+	if t.AdminChannelID != "" {
+		err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", t.AdminChannelID), msg)
+		if err != nil {
+			result = multierror.Append(result,
+				fmt.Errorf("problem sending admin telegram notification about comment ID %s to %s: %w",
+					req.Comment.ID, t.AdminChannelID, err,
+				),
+			)
 		}
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return errors.Errorf("unexpected telegram status code %d for url %q", resp.StatusCode, u)
 	}
 
-	tgResp := struct {
-		OK bool `json:"ok"`
-	}{}
-
-	if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
-		return errors.Wrap(err, "can't decode telegram response")
+	if t.UserNotifications {
+		for _, user := range req.Telegrams {
+			err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", user), msg)
+			if err != nil {
+				result = multierror.Append(result,
+					fmt.Errorf("problem sending user telegram notification about comment ID %s to %q: %w",
+						req.Comment.ID, user, err,
+					),
+				)
+			}
+		}
 	}
+	return result.ErrorOrNil()
+}
+
+// buildMessage generates message for generic notification about new comment
+func (t *Telegram) buildMessage(req Request) string {
+	commentURLPrefix := req.Comment.Locator.URL + uiNav
+
+	msg := fmt.Sprintf(`<a href=%q>%s</a>`, commentURLPrefix+req.Comment.ID, ntf.EscapeTelegramText(req.Comment.User.Name))
+
+	if req.Comment.ParentID != "" {
+		msg += fmt.Sprintf(" -> <a href=%q>%s</a>", commentURLPrefix+req.parent.ID, ntf.EscapeTelegramText(req.parent.User.Name))
+	}
+
+	msg += fmt.Sprintf("\n\n%s", ntf.TelegramSupportedHTML(req.Comment.Text))
+
+	if req.Comment.ParentID != "" {
+		msg += fmt.Sprintf("\n\n\"<i>%s</i>\"", ntf.TelegramSupportedHTML(req.parent.Text))
+	}
+
+	if req.Comment.PostTitle != "" {
+		msg += fmt.Sprintf("\n\n↦  <a href=%q>%s</a>", req.Comment.Locator.URL, ntf.EscapeTelegramText(req.Comment.PostTitle))
+	}
+
+	return msg
+}
+
+// SendVerification is not needed for telegram
+func (t *Telegram) SendVerification(_ context.Context, _ VerificationRequest) error {
 	return nil
 }
 
 func (t *Telegram) String() string {
-	return "telegram: " + t.channelID
+	result := t.Telegram.String()
+	if t.AdminChannelID != "" {
+		result += " with admin notifications to " + t.AdminChannelID
+	}
+	if t.UserNotifications {
+		result += " with user notifications enabled"
+	}
+	return result
 }

backend/app/notify/telegram_test.go

@@ -2,119 +2,61 @@ package notify
 
 import (
 	"context"
-	"net/http"
-	"net/http/httptest"
 	"testing"
-	"time"
 
-	"github.com/go-chi/chi"
+	ntf "github.com/go-pkgz/notify"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store"
 )
 
-func TestTelegram_New(t *testing.T) {
-
-	ts := mockTelegramServer()
-	defer ts.Close()
-
-	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NoError(t, err)
-	assert.NotNil(t, tb)
-	assert.Equal(t, "@remark_test", tb.channelID, "@ added")
-
-	st := time.Now()
-	_, err = NewTelegram("bad-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.EqualError(t, err, "unexpected telegram response {OK:false Result:{FirstName:comments_test ID:707381019 IsBot:false UserName:remark42_test_bot}}")
-	assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
-
-	_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err)
-	assert.Contains(t, err.Error(), "can't decode response:")
-
-	_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.EqualError(t, err, "unexpected telegram status code 404")
-
-	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
-	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
-
-	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
-	assert.Error(t, err, "empty api url not allowed")
-
-	_, err = NewTelegram("good-token", "remark_test", 0, ts.URL+"/")
-	assert.NoError(t, err, "0 timeout allowed as default")
-
-	tb, err = NewTelegram("good-token", "1234567890", 2*time.Second, ts.URL+"/")
-	assert.NoError(t, err)
-	assert.NotNil(t, tb)
-	assert.Equal(t, "1234567890", tb.channelID, "no @ prefix")
+func TestTelegram_NewError(t *testing.T) {
+	tb, err := NewTelegram(TelegramParams{})
+	assert.Error(t, err)
+	assert.Nil(t, tb)
 }
 
 func TestTelegram_Send(t *testing.T) {
-	ts := mockTelegramServer()
-	defer ts.Close()
-
-	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NoError(t, err)
-	assert.NotNil(t, tb)
-	c := store.Comment{Text: "some text", ParentID: "1"}
+	tb := Telegram{
+		AdminChannelID:    "remark_test",
+		UserNotifications: true,
+		Telegram:          &ntf.Telegram{}, // broken sender due to unset API
+	}
+	assert.Equal(t, "telegram notifications destination with admin notifications to remark_test with user notifications enabled", tb.String())
+	c := store.Comment{Text: "some text", ParentID: "1", ID: "999", PostTitle: "[test title]", Locator: store.Locator{URL: "http://example.org/"}}
 	c.User.Name = "from"
-	cp := store.Comment{Text: "some parent text"}
+	cp := store.Comment{Text: `<p>some parent text with a <a href="http://example.org">link</a> and special text:<br>& < > &</p>`}
 	cp.User.Name = "to"
 
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
-	assert.NoError(t, err)
-	c.PostTitle = "test title"
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
-	assert.NoError(t, err)
+	err := tb.Send(context.Background(), Request{Comment: c, parent: cp, Telegrams: []string{"test_user_channel"}})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "2 errors occurred")
+	assert.Contains(t, err.Error(), "problem sending user telegram notification about comment ID 999 to \"test_user_channel\"")
+	assert.Contains(t, err.Error(), "problem sending admin telegram notification about comment ID 999 to remark_test")
 
-	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
-	assert.NotNil(t, err, "should failed")
-	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
-	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
+	// test buildMessage separately for message text
+	res := tb.buildMessage(Request{Comment: c, parent: cp})
+	assert.Equal(t, `<a href="http://example.org/#remark42__comment-999">from</a> -> <a href="http://example.org/#remark42__comment-">to</a>
 
-	assert.Equal(t, "telegram: @remark_test", tb.String())
+some text
+
+"<i>some parent text with a <a href="http://example.org">link</a> and special text:&amp; &lt; &gt; &amp;</i>"
+
+↦  <a href="http://example.org/">[test title]</a>`,
+		res)
+
+	// special case for text with h1-h6 header
+	ch := store.Comment{Text: "<h1>Hello</h1><h6>World</h6>", ID: "555", Locator: store.Locator{URL: "http://example.org/"}}
+	ch.User.Name = "from"
+	res = tb.buildMessage(Request{Comment: ch})
+	assert.Equal(t, `<a href="http://example.org/#remark42__comment-555">from</a>
+
+<b>Hello</b><i><b>World</b></i>`,
+		res)
 }
 
-func mockTelegramServer() *httptest.Server {
-	router := chi.NewRouter()
-	router.Get("/good-token/getMe", func(w http.ResponseWriter, r *http.Request) {
-		s := `{"ok": true,
-				"result": {
-					"first_name": "comments_test",
-					"id": 707381019,
-					"is_bot": true,
-					"username": "remark42_test_bot"
-				}}`
-		_, _ = w.Write([]byte(s))
-	})
-	router.Get("/bad-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
-		s := `{"ok": false,
-				"result": {
-					"first_name": "comments_test",
-					"id": 707381019,
-					"is_bot": false,
-					"username": "remark42_test_bot"
-				}}`
-		_, _ = w.Write([]byte(s))
-	})
-	router.Get("/non-json-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
-		s := `"ok": false,
-				"result": {
-					"first_name": "comments_test",
-					"id": 707381019,
-					"is_bot": false,
-					"username": "remark42_test_bot"
-				`
-		_, _ = w.Write([]byte(s))
-	})
-	router.Get("/404/getMe", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(404)
-	})
-
-	router.Post("/good-token/sendMessage", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = w.Write([]byte(`{"ok": true}`))
-	})
-
-	return httptest.NewServer(router)
+func TestTelegram_SendVerification(t *testing.T) {
+	tb := Telegram{}
+	// empty VerificationRequest should return no error and do nothing, as well as any other
+	assert.NoError(t, tb.SendVerification(context.Background(), VerificationRequest{}))
 }

backend/app/notify/testdata/bad.html.tmpl

@@ -0,0 +1 @@
+{{

backend/app/notify/testdata/msg.html.tmpl

@@ -0,0 +1,20 @@
+{{- if .ForAdmin}}
+New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
+{{- else }}
+	New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
+{{- end }}
+{{- if .ParentCommentText}}
+	{{.ParentUserPicture}}
+	{{.ParentUserName}}
+	{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}
+	Parent comment link: {{.ParentCommentLink}}
+	{{.ParentCommentText}}
+{{- end }}
+
+User: {{.UserName}}
+{{.CommentDate.Format "02.01.2006 at 15:04"}}
+Comment: {{.CommentText}}
+{{.Email}} {{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}
+{{- if .UnsubscribeLink}}
+Unsubscribe link: {{.UnsubscribeLink}}
+{{- end }}

backend/app/notify/testdata/verification.html.tmpl

@@ -0,0 +1,7 @@
+Confirmation for {{.User}} on site {{.Site}}
+{{- if .SubscribeURL}}
+Subscribe url: {{.SubscribeURL}}{{.Token}}
+{{- end }}
+Token:{{.Token}}
+Sent to {{.Email}}
+

backend/app/notify/webhook.go

@@ -0,0 +1,84 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	ntf "github.com/go-pkgz/notify"
+)
+
+const (
+	webhookDefaultTemplate = `{"text": "{{.Text}}"}`
+)
+
+// WebhookParams contain settings for webhook notifications
+type WebhookParams struct {
+	URL      string
+	Template string
+	Headers  []string
+	Timeout  time.Duration
+}
+
+// Webhook implements notify.Destination for Webhook notifications
+type Webhook struct {
+	*ntf.Webhook
+
+	url      string
+	template *template.Template
+}
+
+// NewWebhook makes Webhook
+func NewWebhook(params WebhookParams) (*Webhook, error) {
+	res := &Webhook{
+		Webhook: ntf.NewWebhook(ntf.WebhookParams{
+			Timeout: params.Timeout,
+			Headers: params.Headers,
+		}),
+		url: params.URL,
+	}
+
+	if res.url == "" {
+		return nil, fmt.Errorf("webhook URL is required for webhook notifications")
+	}
+
+	if params.Template == "" {
+		params.Template = webhookDefaultTemplate
+	}
+
+	payloadTmpl, err := template.New("webhook").Parse(params.Template)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse webhook template: %w", err)
+	}
+
+	res.template = payloadTmpl
+
+	log.Printf("[DEBUG] create new webhook notifier for %s", res.url)
+
+	return res, nil
+}
+
+// Send sends Webhook notification
+func (w *Webhook) Send(ctx context.Context, req Request) error {
+	log.Printf("[DEBUG] send webhook notification, comment id %s", req.Comment.ID)
+	var payload bytes.Buffer
+	err := w.template.Execute(&payload, req.Comment)
+	if err != nil {
+		return fmt.Errorf("unable to compile webhook template: %w", err)
+	}
+
+	return w.Webhook.Send(ctx, w.url, payload.String())
+}
+
+// SendVerification is not implemented for Webhook
+func (w *Webhook) SendVerification(_ context.Context, _ VerificationRequest) error {
+	return nil
+}
+
+// String describes the webhook instance
+func (w *Webhook) String() string {
+	return fmt.Sprintf("%s to %s", w.Webhook.String(), w.url)
+}

backend/app/notify/webhook_test.go

@@ -0,0 +1,89 @@
+package notify
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+)
+
+func TestWebhook_NewWebhook(t *testing.T) {
+	wh, err := NewWebhook(WebhookParams{
+		URL:     "https://example.org/webhook",
+		Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	assert.Equal(t, "https://example.org/webhook", wh.url)
+	assert.Equal(t, []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, wh.Headers)
+	assert.NotNil(t, wh.template)
+
+	wh, err = NewWebhook(WebhookParams{})
+	assert.Nil(t, wh)
+	assert.Error(t, err)
+	assert.Equal(t, "webhook URL is required for webhook notifications", err.Error())
+
+	wh, err = NewWebhook(WebhookParams{URL: "https://example.org/webhook", Template: "{{.Text"})
+	assert.Nil(t, wh)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unable to parse webhook template")
+}
+
+func TestWebhook_Send(t *testing.T) {
+	wh, err := NewWebhook(WebhookParams{
+		URL:     "bad-url",
+		Headers: []string{"Content-Type:application/json,text/plain", ""},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
+	c.User.Name = "from"
+
+	err = wh.Send(context.Background(), Request{Comment: c})
+	assert.Error(t, err)
+
+	wh, err = NewWebhook(WebhookParams{
+		URL:      "https://example.org/webhook",
+		Template: "{{.InvalidProperty}}",
+	})
+	assert.NoError(t, err)
+	err = wh.Send(context.Background(), Request{Comment: c})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "webhook template")
+
+	wh, err = NewWebhook(WebhookParams{URL: "https://example.org/webhook"})
+	assert.NoError(t, err)
+	err = wh.Send(nil, Request{Comment: c}) // nolint
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unable to create webhook request")
+
+	wh, err = NewWebhook(WebhookParams{URL: "https://not-existing-url.net"})
+	assert.NoError(t, err)
+	err = wh.Send(context.Background(), Request{Comment: c})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "webhook request failed")
+}
+
+func TestWebhook_SendVerification(t *testing.T) {
+	wh, err := NewWebhook(WebhookParams{URL: "https://example.org/webhook"})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	err = wh.SendVerification(context.Background(), VerificationRequest{})
+	assert.NoError(t, err)
+}
+
+func TestWebhook_String(t *testing.T) {
+	wh, err := NewWebhook(WebhookParams{URL: "https://example.org/webhook", Timeout: time.Minute * 5})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	str := wh.String()
+	assert.Equal(t, "webhook notification with timeout 5m0s to https://example.org/webhook", str)
+}

backend/app/providers/telegram.go

@@ -0,0 +1,71 @@
+package providers
+
+// Both Telegram auth and notifications need to receive messages received by Telegram bot in the loop,
+// and below is the implementation of such loop which dispatched received events to both receivers,
+// so that they could work at the same time.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	ntf "github.com/go-pkgz/notify"
+)
+
+type tgRequester interface {
+	Request(ctx context.Context, method string, b []byte, data interface{}) error
+}
+
+// TGUpdatesReceiver used to dispatch telegram updates to multiple receivers
+type TGUpdatesReceiver interface {
+	fmt.Stringer
+	ProcessUpdate(ctx context.Context, textUpdate string) error
+}
+
+// DispatchTelegramUpdates dispatches telegram updates to provided list of receivers
+// Blocks caller
+func DispatchTelegramUpdates(ctx context.Context, requester tgRequester, receivers []TGUpdatesReceiver, period time.Duration) {
+	// Identifier of the first update to be requested.
+	// Should be equal to LastSeenUpdateID + 1
+	// See https://core.telegram.org/bots/api#getupdates
+	var updateOffset int
+
+	processUpdatedTicker := time.NewTicker(period)
+	for {
+		select {
+		case <-ctx.Done():
+			processUpdatedTicker.Stop()
+			return
+		case <-processUpdatedTicker.C:
+			url := `getUpdates?allowed_updates=["message"]`
+			if updateOffset != 0 {
+				url += fmt.Sprintf("&offset=%d", updateOffset)
+			}
+
+			var update ntf.TelegramUpdate
+
+			err := requester.Request(ctx, url, nil, &update)
+			if err != nil {
+				log.Printf("[WARN] failed to fetch updates: %v", err)
+				continue
+			}
+
+			for _, u := range update.Result {
+				if u.UpdateID >= updateOffset {
+					updateOffset = u.UpdateID + 1
+				}
+			}
+
+			if raw, err := json.Marshal(update); err == nil {
+				for _, r := range receivers {
+					e := r.ProcessUpdate(ctx, string(raw))
+					if e != nil {
+						log.Printf("[ERROR] failure from destination %s on processing telegram update %v", r, e)
+					}
+				}
+			}
+		}
+	}
+}

backend/app/providers/telegram_test.go

@@ -0,0 +1,71 @@
+package providers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	ntf "github.com/go-pkgz/notify"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDispatchTelegramUpdates(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	poolPeriod := time.Millisecond * 100
+	go DispatchTelegramUpdates(ctx, &mockTGRequester{t: t}, []TGUpdatesReceiver{&mockTGUpdatesReceiver{t: t}}, poolPeriod)
+	time.Sleep(poolPeriod * 3)
+	cancel()
+	time.Sleep(poolPeriod)
+}
+
+const getUpdatesResp = `{
+  "ok": true,
+  "result": [
+     {
+        "update_id": 998,
+        "message": {
+           "chat": {
+              "type": "group"
+           }
+        }
+     }
+]
+}`
+
+type mockTGRequester struct {
+	hit int
+	t   *testing.T
+}
+
+func (m *mockTGRequester) Request(_ context.Context, _ string, _ []byte, data interface{}) error {
+	if m.hit < 2 {
+		m.hit++
+		assert.NoError(m.t, json.Unmarshal([]byte(getUpdatesResp), data))
+		return nil
+	}
+	return fmt.Errorf("test error")
+}
+
+type mockTGUpdatesReceiver struct {
+	t   *testing.T
+	hit int
+}
+
+func (m *mockTGUpdatesReceiver) String() string {
+	return "mock updater"
+}
+
+func (m *mockTGUpdatesReceiver) ProcessUpdate(_ context.Context, textUpdate string) error {
+	var result ntf.TelegramUpdate
+	err := json.Unmarshal([]byte(textUpdate), &result)
+	assert.NoError(m.t, err)
+	if m.hit < 2 {
+		assert.NotNil(m.t, result.Result)
+		m.hit++
+		return nil
+	}
+	assert.Nil(m.t, result.Result)
+	return fmt.Errorf("test error")
+}

backend/app/rest/api/admin.go

@@ -1,77 +1,71 @@
 package api
 
 import (
-	"errors"
+	"fmt"
 	"net/http"
 	"path"
 	"time"
 
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
+	cache "github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/go-pkgz/rest/cache"
 
-	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/umputun/remark42/backend/app/rest"
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService   *service.DataStore
-	cache         cache.LoadingCache
+	dataService   adminStore
+	cache         LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
+type adminStore interface {
+	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
+	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
+	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	IsBlocked(siteID string, userID string) bool
+	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
+	BlockedUsers(siteID string) ([]store.BlockedUser, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
+	SetVerified(siteID string, userID string, status bool) error
+	SetReadOnly(locator store.Locator, status bool) error
+	SetPin(locator store.Locator, commentID string, status bool) error
 }
 
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
 func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
 	id := chi.URLParam(r, "id")
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[INFO] delete comment %s", id)
 
 	err := a.dataService.Delete(locator, id, store.SoftDelete)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment", rest.ErrInternal)
 		return
 	}
-	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
 
 // DELETE /user/{userid}?site=side-id - delete all user comments for requested userid
 func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
-
 	userID := chi.URLParam(r, "userid")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
+	if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -81,29 +75,27 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 
 // GET /user/{userid}?site=side-id - get user info for requested userid
 func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
-
 	userID := chi.URLParam(r, "userid")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
 
-	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0, rest.GetUserOrEmpty(r))
 	if err != nil || len(ucomments) == 0 {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
 	}
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, ucomments[0].User)
 }
 
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
+// GET /deleteme?token=jwt - delete all user comments and details by user's request. Gets info about deleted used from provided token
 // request made GET to allow direct click from the email sent by user
 func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
-
 	token := r.URL.Query().Get("token")
 
 	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token", rest.ErrActionRejected)
 		return
 	}
 
@@ -111,19 +103,25 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
 	if !claims.User.BoolAttr("delete_me") {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, fmt.Errorf("forbidden"), "can't use provided token", rest.ErrNoAccess)
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
+	if err = a.dataService.DeleteUserDetail(claims.Audience, claims.User.ID, engine.AllUserDetails); err != nil {
+		code := parseError(err, rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user details for user", code)
+		return
+	}
+
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+		avatarStore := a.authenticator.AvatarProxy().Store
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
 	}
@@ -147,9 +145,16 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
+
+	// delete comments for permanently blocked user.
+	if blockStatus && ttl == time.Duration(0) {
+		if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
+			log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
+		}
+	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
@@ -157,9 +162,9 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 // GET /blocked?site=siteID - list blocked users
 func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
+	users, err := a.dataService.BlockedUsers(siteID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
 	}
 	render.JSON(w, r, users)
@@ -178,13 +183,14 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
 	if !roStatus {
 		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
+			rest.SendErrorJSON(w, r, http.StatusForbidden, fmt.Errorf("rejected"),
+				"read-only due the age", rest.ErrActionRejected)
 			return
 		}
 	}
 
 	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status", rest.ErrPostNotFound)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
@@ -198,7 +204,7 @@ func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
 
 	c, err := a.dataService.SetTitle(locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title", rest.ErrInternal)
 		return
 	}
 	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
@@ -215,7 +221,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	verifyStatus := r.URL.Query().Get("verified") == "1"
 
 	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
@@ -230,48 +236,9 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	pinStatus := r.URL.Query().Get("pin") == "1"
 
 	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}