add escaping of comment text in webhook default JSON template

A check in image proxy for validating content type of requested images added. Modified the related tests to accommodate these changes.

.dockerignore

@@ -1,15 +1,30 @@
 /logs/
 /target/
 /var/
-/frontend/node_modules/
-/frontend/public/
 /.vscode/
 /.idea/
 /bin/
+/.git/
+
+# frontend files not needed in docker image
+/frontend/node_modules/
+/frontend/apps/remark42/node_modules/
+/frontend/apps/remark42/public/
+# e2e tests arficats
+/frontend/e2e/playwright-report/
+/frontend/e2e/playwright/.cache/
+/frontend/e2e/test-results/
 
 # source files
 docker-compose.yml
+compose-dev-backend.yml
+compose-dev-frontend.yml
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-e2e-test.yml
+compose-private.yml
 rest-client.env.json
+Makefile
 
 # generated files
 *.cov
@@ -21,4 +36,4 @@ debug.test
 *.test
 remark42
 /backend/var/
-compose-private-backend.yml
+/playwright-report/

.editorconfig

@@ -1,3 +1,5 @@
+root = true
+
 [*]
 indent_style = tab
 insert_final_newline = true

.github/CODEOWNERS

@@ -2,4 +2,5 @@
 # Unless a later match takes precedence, @umputun will be requested for
 # review when someone opens a pull request.
 
-*       @umputun
+*             @umputun
+frontend/*    @akellbl4 @Mavrin

.github/workflows/ci-backend.yml

@@ -1,4 +1,4 @@
-name: test_backend
+name: backend
 
 on:
   push:
@@ -10,17 +10,19 @@ on:
       - "!backend/scripts/**"
       - "!**.md"
   pull_request:
+    types: [opened, reopened]
     paths:
       - ".github/workflows/ci-test-backend.yml"
       - "backend/**"
       - "!backend/scripts/**"
       - "!**.md"
 jobs:
-  test-backend:
+  test:
+    name: Test & Coverage
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: debug if needed
         run: if [[ "$DEBUG" == "true" ]]; then env; fi
@@ -28,37 +30,44 @@ jobs:
           DEBUG: ${{secrets.DEBUG}}
 
       - name: install go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: "1.21"
 
-      - name: install golangci-lint and goveralls
-        run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.44.0
-          go get -u github.com/mattn/goveralls
-
-      - name: test and lint backend
+      - name: test and build backend
         run: |
           go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
           cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/app
         env:
-          GOFLAGS: "-mod=vendor"
           TZ: "America/Chicago"
 
-      - name: test and lint examples
+      - name: test examples
         run: |
-          go version
-          $GITHUB_WORKSPACE/golangci-lint version
           go test -race ./...
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/_example/memory_store
         env:
           TZ: "America/Chicago"
 
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          working-directory: backend/app
+
+      - name: golangci-lint on example directory
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          args: --config ../../.golangci.yml
+          working-directory: backend/_example/memory_store
+
       - name: submit coverage
-        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        run: |
+          go install github.com/mattn/goveralls@latest
+          goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
         working-directory: backend
         env:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-build.yml

@@ -7,16 +7,17 @@ on:
     paths:
       - ".github/workflows/ci-build.yml"
       - "backend/**"
-      - "frontend/**"
+      - "frontend/apps/**"
       - ".dockerignore"
       - "docker-init.sh"
       - "Dockerfile"
       - "!**.md"
+      - "!frontend/packages/**"
   pull_request:
     paths:
       - ".github/workflows/ci-build.yml"
       - "backend/**"
-      - "frontend/**"
+      - "frontend/apps/**"
       - ".dockerignore"
       - "docker-init.sh"
       - "Dockerfile"
@@ -24,21 +25,36 @@ on:
 
 jobs:
   build-images:
+    name: Build Docker images
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
+      - name: build docker image without pushing (only outside master)
+        if: ${{ github.ref != 'refs/heads/master' }}
+        run: |
+          docker buildx build \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
+              --platform linux/amd64 .
+
+      - name: build example docker image without pushing (only outside master)
+        if: ${{ github.ref != 'refs/heads/master' }}
+        run: |
+          docker buildx build \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
+              --platform linux/amd64 -f backend/_example/memory_store/Dockerfile .
+
       - name: build and deploy master image to ghcr.io and dockerhub
         if: ${{ github.ref == 'refs/heads/master' }}
         env:
@@ -51,12 +67,12 @@ jobs:
           ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
           echo "GITHUB_REF=${GITHUB_REF}, GITHUB_SHA=${GITHUB_SHA}, GIT_BRANCH=${ref}"
           echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
-          echo ${DOCKER_HUB_TOKEN} | docker login -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u umputun --password-stdin
           docker buildx build --push \
               --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
               --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} --build-arg GITHUB_REF=${GITHUB_REF} \
               --platform linux/amd64,linux/arm/v7,linux/arm64 \
-              -t ghcr.io/${USERNAME}/remark42:${ref} -t ${USERNAME}/remark42:${ref} .
+              -t ghcr.io/umputun/remark42:${ref} -t umputun/remark42:${ref} .
 
       - name: deploy tagged (latest) to ghcr.io and dockerhub
         if: ${{ startsWith(github.ref, 'refs/tags/') }}
@@ -70,7 +86,7 @@ jobs:
           ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
           echo "GITHUB_REF=${GITHUB_REF}, GITHUB_SHA=${GITHUB_SHA}, GIT_BRANCH=${ref}"
           echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
-          echo ${DOCKER_HUB_TOKEN} | docker login -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u umputun --password-stdin
           docker buildx build --push \
               --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
               --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} --build-arg GITHUB_REF=${GITHUB_REF} \

.github/workflows/ci-frontend-api.yml

@@ -0,0 +1,153 @@
+name: "@remark42/api"
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/ci-frontend-api.yml"
+      - "frontend/packages/**"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-frontend-api.yml"
+      - "frontend/packages/**"
+      - "!**.md"
+
+jobs:
+  type-check:
+    name: Type check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.20.2]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run type check
+        run: pnpm type-check:api
+        working-directory: ./frontend
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.20.2]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Run linters
+        run: pnpm lint:api
+        working-directory: ./frontend/
+
+  test:
+    name: Tests & Coverage
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16.20.2]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
+        working-directory: ./frontend
+
+      - name: Test & Coverage
+        run: pnpm coverage:api
+        working-directory: ./frontend
+
+      - name: Submit coverage
+        run: ${{ github.workspace }}/frontend/apps/remark42/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-frontend-size-limit.yml

@@ -1,20 +0,0 @@
-name: frontend
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/ci-frontend-size-limit.yml"
-      - "frontend/**"
-      - "!**.md"
-
-jobs:
-  size:
-    runs-on: ubuntu-latest
-    env:
-      CI_JOB_NUMBER: 1
-    steps:
-      - uses: actions/checkout@v1
-      - uses: andresz1/size-limit-action@v1
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          directory: frontend

.github/workflows/ci-frontend.yml

@@ -3,105 +3,219 @@ name: frontend
 on:
   push:
     branches:
-    tags:
+      - master
     paths:
       - ".github/workflows/ci-frontend.yml"
-      - "frontend/**"
+      - "frontend/apps/remark42/**"
       - "!**.md"
   pull_request:
     paths:
       - ".github/workflows/ci-frontend.yml"
-      - "frontend/**"
+      - "frontend/apps/remark42/**"
       - "!**.md"
 
 jobs:
-  check-translations:
+  translations-check:
+    name: Translations check
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.13.2]
+        node: [16.20.2]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
-      - run: npm ci --loglevel warn
-        working-directory: ./frontend
-
-      - uses: actions/cache@v2
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
         with:
-          path: ${{ github.workspace }}/frontend/node_modules/.cache
-          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          version: 7
+          run_install: false
 
-      - run: npm run check:translation
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
         working-directory: ./frontend
 
-  check-typescript:
+      - name: Translations check
+        run: pnpm translation-check
+        working-directory: ./frontend/apps/remark42
+
+  type-check:
+    name: Type check
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.13.2]
+        node: [16.20.2]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
-      - run: npm ci --loglevel warn
-        working-directory: ./frontend
-
-      - uses: actions/cache@v2
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
         with:
-          path: ${{ github.workspace }}/frontend/node_modules/.cache
-          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          version: 7
+          run_install: false
 
-      - run: npm run check:types
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
         working-directory: ./frontend
 
+      - name: Run type check
+        run: pnpm type-check
+        working-directory: ./frontend/apps/remark42
+
   lint:
+    name: Eslint & Stylelint
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.13.2]
+        node: [16.20.2]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
-      - run: npm ci --loglevel warn
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
         working-directory: ./frontend
 
-      - run: npm run lint
-        working-directory: ./frontend
+      - name: Run linters
+        run: pnpm lint
+        working-directory: ./frontend/apps/remark42
+
+  size-limit:
+    name: Size limit
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Check bundle size
+        uses: andresz1/size-limit-action@7313b26c76b3666c1dc41e2ca05370e201a9b7de
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          directory: ./frontend/apps/remark42
+          package_manager: pnpm
 
   test:
+    name: Tests & Coverage
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.13.2]
+        node: [16.20.2]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install node
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
-      - run: npm ci --loglevel warn
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2.4.0
+        id: pnpm-install
+        with:
+          version: 7
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm i
         working-directory: ./frontend
 
-      - run: npm run test:coverage
-        working-directory: ./frontend
+      - name: Test & Coverage
+        run: pnpm coverage
+        working-directory: ./frontend/apps/remark42
 
-      - name: submit coverage
-        run: node ${{ github.workspace }}/frontend/node_modules/.bin/codecov
+      - name: Submit coverage
+        run: ${{ github.workspace }}/frontend/apps/remark42/node_modules/.bin/codecov
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-site.yml

@@ -3,6 +3,7 @@ name: site
 on:
   push:
     branches:
+      - master
     tags:
     paths:
       - ".github/workflows/ci-site.yml"
@@ -13,20 +14,20 @@ on:
       - "site/**"
 
 jobs:
-
-  build-site:
+  build:
+    name: Build
     runs-on: ubuntu-latest
 
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
@@ -44,7 +45,7 @@ jobs:
           echo GITHUB_REF - $ref
           echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
           docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
-            -t ghcr.io/${USERNAME}/remark24-site:${ref} .
+            -t ghcr.io/umputun/remark42-site:${ref} .
 
       - name: deploy tagged (latest) to ghcr.io and dockerhub
         if: ${{ startsWith(github.ref, 'refs/tags/') }}
@@ -59,7 +60,7 @@ jobs:
           echo "GITHUB_REF=$ref, GITHUB_SHA=${GITHUB_SHA}"
           echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
           docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
-            -t ghcr.io/${USERNAME}/remark24-site:${ref} -t ghcr.io/${USERNAME}/remark24-site:latest .
+            -t ghcr.io/umputun/remark42-site:${ref} -t ghcr.io/umputun/remark42-site:latest .
 
       - name: remote site deployment from master
         if: ${{ github.ref == 'refs/heads/master' }}

.github/workflows/e2e-tests.yml

@@ -0,0 +1,34 @@
+name: e2e
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - "frontend/apps/remark42/**"
+      - "frontend/e2e/**"
+
+  pull_request:
+    branches: [master]
+    paths:
+      - "frontend/apps/remark42/**"
+      - "frontend/e2e/**"
+
+jobs:
+  tests:
+    name: Tests
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build & run containers
+        id: tests
+        run: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: playwright-report
+          path: ./playwright-report/
+          retention-days: 30

.gitignore

@@ -4,7 +4,6 @@ target
 /logs/
 /target/
 /var/
-/web/
 debug
 debug.test
 .vscode
@@ -18,10 +17,12 @@ remark42
 /bin/
 /backend/var/
 /backend/app/var/
-/backend/web/
+/backend/app/cmd/web/
 /backend/*.html.tmpl
 compose-private-backend.yml
 compose-private-frontend.yml
 compose-private.yml
 /backend/_example/*/vendor
 http-client.env.json
+/playwright-report/
+/backend/app/cmd/var

Dockerfile

@@ -1,4 +1,51 @@
-FROM umputun/baseimage:buildgo-v1.8.0 as build-backend
+FROM --platform=$BUILDPLATFORM node:16.20-alpine AS frontend-deps
+
+ARG SKIP_FRONTEND_TEST
+ARG SKIP_FRONTEND_BUILD
+
+WORKDIR /srv/frontend/
+
+COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
+COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/
+
+RUN \
+  if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
+    apk add --no-cache --update git && \
+    npm i -g pnpm@7; \
+  fi
+
+RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 \
+  if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
+    pnpm i; \
+  fi
+
+
+FROM --platform=$BUILDPLATFORM frontend-deps AS build-frontend
+
+ARG SKIP_FRONTEND_TEST
+ARG SKIP_FRONTEND_BUILD
+ENV CI=true
+
+WORKDIR /srv/frontend/apps/remark42/
+
+COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
+
+RUN \
+  if [ -z "$SKIP_FRONTEND_TEST" ]; then \
+    pnpm lint type-check translation-check test; \
+  else \
+    echo 'Skip frontend test'; \
+  fi
+
+RUN \
+  if [ -z "$SKIP_FRONTEND_BUILD" ]; then \
+    pnpm build; \
+  else \
+    mkdir /srv/frontend/apps/remark42/public; \
+    echo 'Skip frontend build'; \
+  fi
+
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG CI
 ARG GITHUB_REF
@@ -8,11 +55,11 @@ ARG SKIP_BACKEND_TEST
 ARG BACKEND_TEST_TIMEOUT
 
 ADD backend /build/backend
-ADD .git/ /build/backend/.git/
+# to embed the frontend files statically into Remark42 binary
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
+RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
 WORKDIR /build/backend
 
-ENV GOFLAGS="-mod=vendor"
-
 # install gcc in order to be able to go test package with -race
 RUN apk --no-cache add gcc libc-dev
 
@@ -23,10 +70,10 @@ RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
         CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
-        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov && \
         golangci-lint run --config ../.golangci.yml ./... ; \
     else \
-    	echo "skip backend tests and linter" \
+      echo "skip backend tests and linter" \
     ; fi
 
 RUN \
@@ -34,65 +81,33 @@ RUN \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM --platform=$BUILDPLATFORM node:16.13.2-alpine as build-frontend-deps
+FROM umputun/baseimage:app-v1.13.0
 
-ARG CI
-ARG SKIP_FRONTEND_BUILD
-ENV HUSKY_SKIP_INSTALL=true
+ARG GITHUB_SHA
 
-RUN if [ -z "$SKIP_FRONTEND_BUILD" ] ; then \
-    	apk add --no-cache --update git \
-    ; fi
-ADD frontend/package.json /srv/frontend/package.json
-ADD frontend/package-lock.json /srv/frontend/package-lock.json
-WORKDIR /srv/frontend
-RUN mkdir node_modules
-RUN if [ -z "$SKIP_FRONTEND_BUILD" ] ; then \
-    	CI=true npm ci --loglevel warn \
-    else \
-    	echo "skip frontend build" \
-    ; fi
-
-FROM --platform=$BUILDPLATFORM node:16.13.2-alpine as build-frontend
-
-ARG CI
-ARG SKIP_FRONTEND_TEST
-ARG SKIP_FRONTEND_BUILD
-ARG NODE_ENV=production
-
-COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD frontend /srv/frontend
-WORKDIR /srv/frontend
-RUN mkdir public
-RUN if [ -z "$SKIP_FRONTEND_BUILD" ] ; then \
-		if [ -z "$SKIP_FRONTEND_TEST" ] ; then \
-			npm run lint test check; \
-			else \
-    			echo "skip frontend tests and lint" ; npm run build \
-    	; fi \
-    ; fi
-RUN rm -rf ./node_modules
-
-FROM umputun/baseimage:app-v1.8.0
+LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
+      org.opencontainers.image.description="Remark42 comment engine" \
+      org.opencontainers.image.documentation="https://remark42.com/docs/getting-started/" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
+      org.opencontainers.image.title="Remark42" \
+      org.opencontainers.image.url="https://remark42.com/" \
+      org.opencontainers.image.revision="${GITHUB_SHA}"
 
 WORKDIR /srv
 
-ADD docker-init.sh /entrypoint.sh
+COPY docker-init.sh /srv/init.sh
 ADD backend/scripts/backup.sh /usr/local/bin/backup
 ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
-RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
+RUN chmod +x /srv/init.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
 COPY --from=build-backend /build/backend/remark42 /srv/remark42
-COPY --from=build-backend /build/backend/templates /srv
-COPY --from=build-frontend /srv/frontend/public/ /srv/web
-COPY docker-init.sh /srv/init.sh
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /srv/web/
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-
-RUN chmod +x /srv/init.sh
 CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -1,53 +1,41 @@
-FROM node:12.16-alpine as build-frontend-deps
+FROM node:16.20.2-alpine AS frontend-deps
 
-ARG CI
+ENV CI=true
 
-ENV SKIP_FRONTEND_TEST=true
+WORKDIR /srv/frontend
 
-RUN apk add --no-cache --update git
-ADD frontend/package.json /srv/frontend/package.json
-ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/frontend && CI=true npm ci
+COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
+COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/package.json
 
-FROM node:12.16-alpine as build-frontend
+RUN apk add --no-cache --update git && npm i -g pnpm@7
+RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 pnpm i
 
-ARG CI
-ARG NODE_ENV=production
-ENV SKIP_FRONTEND_TEST=true
-ENV HUSKY_SKIP_INSTALL=true
+FROM frontend-deps AS build-frontend
 
-COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD frontend /srv/frontend
-RUN cd /srv/frontend && \
-    npm run build && \
-    rm -rf ./node_modules
+ENV NODE_ENV=production
+ENV CI=true
 
-FROM umputun/baseimage:buildgo-latest as build-backend
+WORKDIR /srv/frontend/apps/remark42/
+COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
+RUN pnpm build
+
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG GITHUB_TOKEN
-ENV SKIP_BACKEND_TEST=true
+ARG GITHUB_REF
+ARG GITHUB_SHA
 
 WORKDIR /build/backend
 ADD backend /build/backend
 ADD README.md /build/
 ADD LICENSE /build/
 
-ADD .git/ /build/backend/.git/
+COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
 
-COPY --from=build-frontend /srv/frontend/public/ web
+RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
 
 RUN \
-    export WEB_ROOT=/build/backend/web && \
-    find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \
-    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
-    statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \
-    ls -la /build/backend/app/templates/statik.go && \
-    ls -la /build/backend/app/rest/api/statik.go && \
-    ls -la /build/backend/web/
-
-RUN \
-	version=$("/script/version.sh") && echo "version=${version}" && \
-    export GOFLAGS="-mod=vendor" && \
+  version=$("/script/version.sh") && echo "version=${version}" && \
     GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
     GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \

Makefile

@@ -1,5 +1,7 @@
 OS=linux
 ARCH=amd64
+GITHUB_REF=$(shell git rev-parse --symbolic-full-name HEAD)
+GITHUB_SHA=$(shell git rev-parse --short HEAD)
 
 bin:
 	docker build -f Dockerfile.artifacts -t remark42.bin .
@@ -9,15 +11,18 @@ bin:
 	docker rm -f remark42.bin
 
 docker:
-	docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
+	DOCKER_BUILDKIT=1 docker build -t umputun/remark42 --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) \
+		--build-arg CI=true --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
 
 dockerx:
-	docker buildx build --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
-		  --progress=plain --platform linux/amd64,linux/arm/v7,linux/arm64 \
-		  -t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
+	docker buildx build --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) --build-arg CI=true \
+		--build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
+		--progress=plain --platform linux/amd64,linux/arm/v7,linux/arm64 \
+		-t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
 
 release:
-	docker build -f Dockerfile.artifacts --no-cache --pull -t remark42.bin .
+	docker build -f Dockerfile.artifacts --no-cache --pull --build-arg CI=true \
+		--build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) -t remark42.bin .
 	- @docker rm -f remark42.bin 2>/dev/null || exit 0
 	- @mkdir -p bin
 	docker run -d --name=remark42.bin remark42.bin
@@ -25,12 +30,13 @@ release:
 	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.darwin-arm64.tar.gz bin/remark42.darwin-arm64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
 	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
 	docker rm -f remark42.bin
 
 race_test:
-	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+	cd backend/app && go test -race -timeout=60s -count 1 ./...
 
 backend:
 	docker-compose -f compose-dev-backend.yml build
@@ -39,8 +45,11 @@ frontend:
 	docker-compose -f compose-dev-frontend.yml build
 
 rundev:
-	docker pull umputun/baseimage:buildgo-latest
-	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true GITHUB_REF=$(GITHUB_REF) GITHUB_SHA=$(GITHUB_SHA) CI=true \
+		docker-compose -f compose-private.yml build
 	docker-compose -f compose-private.yml up
 
+e2e:
+	docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
+
 .PHONY: bin backend

README.md

@@ -1,4 +1,4 @@
-# Remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://app.codecov.io/gh/umputun/remark42)
+# Remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Image Size](https://img.shields.io/docker/image-size/umputun/remark42/master)](https://hub.docker.com/r/umputun/remark42) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://app.codecov.io/gh/umputun/remark42)
 
 Remark42 is a self-hosted, lightweight and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles, or any other place where readers add comments.
 
@@ -37,6 +37,12 @@ For admin screenshots see [Admin UI documentation](https://remark42.com/docs/man
 
 All remark42 documentation is available [by the link](https://remark42.com/docs/getting-started/installation/).
 
+## Contribution
+
+In order to start and work on the project locally in development mode check our contribution documentation for [backend](https://remark42.com/docs/contributing/backend/) and [frontend](https://remark42.com/docs/contributing/frontend/).
+
+If you are interested in adding a new localization please check [these docs](https://remark42.com/docs/contributing/translations/).
+
 ## Related projects
 
 * [A Helm chart for Remark42 on Kubernetes](https://github.com/groundhog2k/helm-charts/tree/master/charts/remark42)

backend/.golangci.yml

@@ -1,17 +1,10 @@
 run:
   timeout: 5m
-  output:
-    format: tab
-  skip-dirs:
-    - vendor
 
 linters-settings:
   govet:
-    check-shadowing: true
-  golint:
-    min-confidence: 0.1
-  maligned:
-    suggest-new: true
+    enable:
+      - shadow
   goconst:
     min-len: 2
     min-occurrences: 2
@@ -33,40 +26,38 @@ linters-settings:
 linters:
   enable:
     - bodyclose
-    - megacheck
-    - revive
-    - govet
-    - unconvert
-    - megacheck
-    - structcheck
-    - gas
-    - gocyclo
     - dupl
-    - misspell
-    - unparam
-    - varcheck
-    - deadcode
-    - typecheck
-    - ineffassign
-    - varcheck
-    - stylecheck
-    - gochecknoinits
     - exportloopref
+    - gochecknoinits
     - gocritic
-    - nakedret
+    - gocyclo
+    - gosec
     - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
     - prealloc
+    - revive
+    - staticcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
   fast: false
   disable-all: true
 
 issues:
+  exclude-dirs:
+    - vendor
   exclude-rules:
     - text: "at least one file in a package should have a package comment"
       linters:
         - stylecheck
-    - text: "should have a package comment, unless it's in another file for this package"
+    - text: "package-comments: should have a package comment"
       linters:
-        - golint
+        - revive
     - path: _test\.go
       linters:
         - gosec

backend/README.md

@@ -0,0 +1 @@
+../site/src/docs/contributing/backend/index.md

backend/_example/memory_store/Dockerfile

@@ -1,12 +1,22 @@
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ADD backend /build/backend
 WORKDIR /build/backend/_example/memory_store
 
 RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
 
+FROM umputun/baseimage:app-v1.13.0
 
-FROM umputun/baseimage:app-latest
+ARG GITHUB_SHA
+
+LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
+      org.opencontainers.image.description="Remark42 comment engine example JRPC memory store" \
+      org.opencontainers.image.documentation="https://github.com/umputun/remark42/tree/master/backend/_example/memory_store" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
+      org.opencontainers.image.title="Remark42 JRPC example memory store" \
+      org.opencontainers.image.url="https://remark42.com/" \
+      org.opencontainers.image.revision="${GITHUB_SHA}"
 
 WORKDIR /srv
 COPY --from=build-backend /build/bin/memory_store /srv/memory_store

backend/_example/memory_store/accessor/admin.go

@@ -35,7 +35,8 @@ func NewMemAdminStore(key string) *MemAdmin {
 	return &MemAdmin{data: map[string]AdminRec{}, key: key}
 }
 
-// Key executes find by siteID and returns substructure with secret key
+// Key supposed to execute find by siteID and returns substructure with secret key,
+// but in this case the shared secret is used for all sites
 func (m *MemAdmin) Key(_ string) (key string, err error) {
 	return m.key, nil
 }

backend/_example/memory_store/accessor/data.go

@@ -8,11 +8,11 @@ package accessor
 
 import (
 	"fmt"
-	"log"
 	"sort"
 	"sync"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/engine"
 )
@@ -267,7 +267,7 @@ func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err erro
 		return res, nil
 
 	case engine.Blocked:
-		log.Printf("%+v", m.metaUsers)
+		log.Printf("[INFO] metaUsers: %+v", m.metaUsers)
 		for _, u := range m.metaUsers {
 			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
 				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
@@ -293,17 +293,17 @@ func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailE
 		defer m.mu.Unlock()
 
 		if req.Update == "" { // read detail value, no update requested
-			return m.getUserDetail(req)
+			return m.getUserDetail(req), nil
 		}
 
-		return m.setUserDetail(req)
+		return m.setUserDetail(req), nil
 	case engine.AllUserDetails:
 		// list of all details returned in case request is a read request
 		// (Update is not set) and does not have UserID or Detail set
 		if req.Update == "" && req.UserID == "" { // read list of all details
 			m.mu.Lock()
 			defer m.mu.Unlock()
-			return m.listDetails(req.Locator)
+			return m.listDetails(req.Locator), nil
 		}
 		return nil, fmt.Errorf("unsupported request with userdetail all")
 	default:
@@ -319,7 +319,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 
 	switch {
 	case req.UserDetail != "": // delete user detail
-		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		return nil
 	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
 		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
 
@@ -332,7 +333,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 				return e
 			}
 		}
-		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		return nil
 
 	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
 		if _, ok := m.posts[req.Locator.SiteID]; !ok {
@@ -437,29 +439,29 @@ func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
 
 // getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
 // as an only element of the slice.
-func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		switch req.Detail {
 		case engine.UserEmail:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}
 		case engine.UserTelegram:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}
 		}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
 // element of the slice in case of success
-func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	var entry metaUser
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		entry = meta
 	}
@@ -476,42 +478,42 @@ func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDeta
 	case engine.UserEmail:
 		entry.Details.Email = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}
 	case engine.UserTelegram:
 		entry.Details.Telegram = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // listDetails lists all available users details for given siteID
-func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+func (m *MemData) listDetails(loc store.Locator) []engine.UserDetailEntry {
 	var res []engine.UserDetailEntry
 	for _, u := range m.metaUsers {
 		if u.SiteID == loc.SiteID {
 			res = append(res, u.Details)
 		}
 	}
-	return res, nil
+	return res
 }
 
 // deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
 // deletion of the absent entry doesn't produce error.
 // Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
-func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) {
 	var entry metaUser
 	if meta, ok := m.metaUsers[userID]; ok {
 		if meta.SiteID != locator.SiteID {
-			return nil
+			return
 		}
 		entry = meta
 	}
 
 	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
 		// absent entry means that we should not do anything
-		return nil
+		return
 	}
 
 	switch userDetail {
@@ -529,7 +531,6 @@ func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDet
 	}
 
 	m.metaUsers[userID] = entry
-	return nil
 }
 
 func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {

backend/_example/memory_store/accessor/data_test.go

@@ -547,12 +547,12 @@ func TestMemData_FlagListBlocked(t *testing.T) {
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	var blockedIds = make([]string, len(blockedList))
+	var blockedIDs = make([]string, len(blockedList))
 	for i, x := range blockedList {
-		blockedIds[i] = x.ID
+		blockedIDs[i] = x.ID
 	}
 	require.Equal(t, 2, len(blockedList), b.metaUsers)
-	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIDs)
 	t.Logf("%+v", blockedList)
 
 	// check block expiration
@@ -624,6 +624,7 @@ func TestMemData_DeleteComment(t *testing.T) {
 func TestMemData_Close(t *testing.T) {
 	b := prepMem(t)
 	assert.NoError(t, b.Close())
+	assert.NoError(t, b.Close(), "second call should not result in panic or errors")
 }
 
 func TestMemData_DeleteHard(t *testing.T) {

backend/_example/memory_store/accessor/image.go

@@ -70,6 +70,17 @@ func (m *MemImage) Load(id string) ([]byte, error) {
 	return img, nil
 }
 
+// Delete image by ID
+func (m *MemImage) Delete(id string) error {
+	m.mu.Lock()
+	// delete key from permanent and staging storage
+	delete(m.images, id)
+	delete(m.insertTime, id)
+	delete(m.imagesStaging, id)
+	m.mu.Unlock()
+	return nil
+}
+
 // Commit moves image from staging to permanent
 func (m *MemImage) Commit(id string) error {
 	m.mu.RLock()

backend/_example/memory_store/accessor/image_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 // gopher png for test, from https://golang.org/src/image/png/example_test.go
-const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+const rawGopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
 	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
 	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
 	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
@@ -38,7 +38,9 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
 	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
 	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
-func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNG() io.Reader {
+	return base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawGopher))
+}
 
 func TestMemImage_LoadAfterSave(t *testing.T) {
 	svc := NewMemImageStore()
@@ -57,7 +59,8 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, gopher, img)
 
-	svc.ResetCleanupTimer(id)
+	err = svc.ResetCleanupTimer(id)
+	assert.NoError(t, err)
 
 	err = svc.Commit(id)
 	assert.NoError(t, err)
@@ -70,6 +73,26 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.Equal(t, gopher, img)
 }
 
+func TestMemImage_LoadAfterDelete(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	img, err := svc.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+	assert.Empty(t, img)
+
+	err = svc.ResetCleanupTimer(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
 func TestMemImage_CommitFail(t *testing.T) {
 	svc := NewMemImageStore()
 	err := svc.Commit("test_id")

backend/_example/memory_store/go.mod

@@ -1,44 +1,43 @@
 module github.com/umputun/remark42/memory_store
 
-go 1.17
+go 1.21
 
 require (
-	github.com/go-pkgz/jrpc v0.2.0
-	github.com/go-pkgz/lgr v0.10.4
+	github.com/go-pkgz/jrpc v0.3.0
+	github.com/go-pkgz/lgr v0.11.1
 	github.com/jessevdk/go-flags v1.5.0
-	github.com/stretchr/testify v1.7.1
-	github.com/umputun/remark42/backend v1.9.0
+	github.com/stretchr/testify v1.9.0
+	github.com/umputun/remark42/backend v1.1000.0
 )
 
 require (
-	github.com/Depado/bfchroma v1.3.0 // indirect
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
-	github.com/alecthomas/chroma v0.10.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/Depado/bfchroma/v2 v2.0.0 // indirect
+	github.com/PuerkitoBio/goquery v1.9.1 // indirect
+	github.com/ajg/form v1.5.1 // indirect
+	github.com/alecthomas/chroma/v2 v2.13.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/didip/tollbooth/v6 v6.1.2 // indirect
-	github.com/didip/tollbooth_chi v0.0.0-20220429013743-da966f2f674b // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/go-chi/chi v4.1.1+incompatible // indirect
-	github.com/go-chi/render v1.0.1 // indirect
-	github.com/go-pkgz/expirable-cache v0.0.3 // indirect
-	github.com/go-pkgz/rest v1.15.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/didip/tollbooth/v7 v7.0.1 // indirect
+	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/go-chi/chi/v5 v5.0.12 // indirect
+	github.com/go-chi/render v1.0.3 // indirect
+	github.com/go-pkgz/expirable-cache v1.0.0 // indirect
+	github.com/go-pkgz/rest v1.19.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rs/xid v1.4.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 // indirect
-	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
-	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
+	golang.org/x/image v0.15.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/umputun/remark42/backend => ../../
+replace github.com/umputun/remark42/backend v1.1000.0 => ../../

backend/_example/memory_store/go.sum

@@ -1,52 +1,54 @@
-github.com/Depado/bfchroma v1.3.0 h1:zz14vpvySU6S0CL6yGPr1vkFevQecIt8dJdCsMS2JpM=
-github.com/Depado/bfchroma v1.3.0/go.mod h1:c0bFk0tFmT+clD3TIGurjWCfD/QV8/EebfM3JGr+98M=
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
-github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
-github.com/alecthomas/chroma v0.7.3/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM=
-github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
-github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
-github.com/alecthomas/kong v0.2.4/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
-github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/Depado/bfchroma/v2 v2.0.0 h1:IRpN9BPkNwEpR6w1ectIcNWOuhDSLx+8f1pn83fzxx8=
+github.com/Depado/bfchroma/v2 v2.0.0/go.mod h1:wFwW/Pw8Tnd0irzgO9Zxtxgzp3aPS8qBWlyadxujxmw=
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
+github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
+github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
-github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/didip/tollbooth/v6 v6.0.1/go.mod h1:j2pKs+JQ5PvU/K4jFnrnwntrmfUbYLJE5oSdxR37FD0=
-github.com/didip/tollbooth/v6 v6.1.2 h1:Kdqxmqw9YTv0uKajBUiWQg+GURL/k4vy9gmLCL01PjQ=
-github.com/didip/tollbooth/v6 v6.1.2/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s=
-github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
-github.com/didip/tollbooth_chi v0.0.0-20220429013743-da966f2f674b h1:elkngQhLBcyDIXwL9Z7AfXXbBszUEsLxqNw6WPF8Mtc=
-github.com/didip/tollbooth_chi v0.0.0-20220429013743-da966f2f674b/go.mod h1:0ZVa6kSzS011nfTC1rELyxK4tjVf6vqBnOv7oY2KlsA=
-github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
-github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
-github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/didip/tollbooth/v7 v7.0.0/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
+github.com/didip/tollbooth/v7 v7.0.1 h1:TkT4sBKoQoHQFPf7blQ54iHrZiTDnr8TceU+MulVAog=
+github.com/didip/tollbooth/v7 v7.0.1/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
+github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f h1:jtKwihcLmUC9BAhoJ9adCUqdSSZcOdH2KL7mPTUm2aw=
+github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f/go.mod h1:q9C80dnsuVRP2dAskjnXRNWdUJqtGgwG9wNrzt0019s=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
+github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
-github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
-github.com/go-pkgz/jrpc v0.2.0 h1:CLy/eZyekjraVrxZV18N2R1mYLMJ/nWrgdfyIOGPY/E=
-github.com/go-pkgz/jrpc v0.2.0/go.mod h1:wd8vtQ4CgtCnuqua6x2b1SKIgv0VSOh5Dn0uUITbiUE=
-github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
-github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
-github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
-github.com/go-pkgz/rest v1.15.0 h1:v/BDqJF9robo85GME85GWJ7O/NjtAO0x7LvO4EqWNRE=
-github.com/go-pkgz/rest v1.15.0/go.mod h1:KUWAqbDteYGS/CiXftomQsKjtEOifXsJ36Ka0skYbmk=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
+github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
+github.com/go-pkgz/expirable-cache v0.1.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
+github.com/go-pkgz/expirable-cache v1.0.0 h1:ns5+1hjY8hntGv8bPaQd9Gr7Jyo+Uw5SLyII40aQdtA=
+github.com/go-pkgz/expirable-cache v1.0.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
+github.com/go-pkgz/jrpc v0.3.0 h1:Fls38KqPsHzvp0FWfivr6cGnncC+iFBodHBqvUPY+0U=
+github.com/go-pkgz/jrpc v0.3.0/go.mod h1:MFtKs75JESiSqVicsQkgN2iDFFuCd3gVT1/vKiwRi00=
+github.com/go-pkgz/lgr v0.11.1 h1:hXFhZcznehI6imLhEa379oMOKFz7TQUmisAqb3oLOSM=
+github.com/go-pkgz/lgr v0.11.1/go.mod h1:tgDF4RXQnBfIgJqjgkv0yOeTQ3F1yewWIZkpUhHnAkU=
+github.com/go-pkgz/rest v1.15.6/go.mod h1:KUWAqbDteYGS/CiXftomQsKjtEOifXsJ36Ka0skYbmk=
+github.com/go-pkgz/rest v1.19.0 h1:FNMi5QX5dDIkuC+/e0r+CWsTuOTwUiWMRSA16Ou+9+A=
+github.com/go-pkgz/rest v1.19.0/go.mod h1:Po+W6zQzpMPP6XDGLdAN2aW7UKk1IyrLSb48Lp1N3oQ=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -54,65 +56,71 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
-github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
+github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
-github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
+go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 h1:LRtI4W37N+KFebI/qV0OFiLUv4GLOWeEW5hn/KEJvxE=
-golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
-golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
+golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/main.go

@@ -43,16 +43,14 @@ func main() {
 	adminStore := accessor.NewMemAdminStore(opts.Secret)
 	imgStore := accessor.NewMemImageStore()
 
-	rpcServer := jrpc.Server{
-		API:        opts.API,
-		AuthUser:   opts.AuthUser,
-		AuthPasswd: opts.AuthPasswd,
-		Version:    revision,
-		AppName:    "remark42-memory",
-		Logger:     log.Default(),
-	}
+	rpcServer := jrpc.NewServer(
+		opts.API,
+		jrpc.Auth(opts.AuthUser, opts.AuthPasswd),
+		jrpc.WithSignature("remark42-memory", "umputun", revision),
+		jrpc.WithLogger(log.Default()),
+	)
 
-	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
+	srv := server.NewRPC(dataStore, adminStore, imgStore, rpcServer)
 
 	admRec := accessor.AdminRec{
 		SiteID:  "remark",

backend/_example/memory_store/server/data_test.go

@@ -198,25 +198,62 @@ func TestRPC_listFlagsHndl(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "123456", id)
 
-	flagReq := engine.FlagRequest{
+	// verify user
+	verifyFlagReq := engine.FlagRequest{
 		Flag:   engine.Verified,
 		UserID: "u1",
 		Locator: store.Locator{
 			SiteID: "test-site",
 		},
 	}
-	flags, err := re.ListFlags(flagReq)
+	flags, err := re.ListFlags(verifyFlagReq)
 	require.NoError(t, err)
-	assert.Equal(t, []interface{}{}, flags)
+	assert.Empty(t, flags)
 
-	flagReq.Update = engine.FlagTrue
-	status, err := re.Flag(flagReq)
+	verifyFlagReq.Update = engine.FlagTrue
+	status, err := re.Flag(verifyFlagReq)
 	require.NoError(t, err)
 	assert.Equal(t, true, status)
 
-	flags, err = re.ListFlags(flagReq)
+	flags, err = re.ListFlags(verifyFlagReq)
 	require.NoError(t, err)
 	assert.Equal(t, []interface{}{"u1"}, flags)
+	verifiedUsers := make([]string, 0, len(flags))
+	for _, v := range flags {
+		verifiedUsers = append(verifiedUsers, v.(string))
+	}
+	assert.Equal(t, []string{"u1"}, verifiedUsers)
+
+	// block user
+	blockFlagReq := engine.FlagRequest{
+		Flag:   engine.Blocked,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		TTL: time.Hour,
+	}
+	flags, err = re.ListFlags(blockFlagReq)
+	require.NoError(t, err)
+	assert.Empty(t, flags)
+
+	blockFlagReq.Update = engine.FlagTrue
+	status, err = re.Flag(blockFlagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(blockFlagReq)
+	require.NoError(t, err)
+	assert.NotEmpty(t, flags)
+	blockedUsers := make([]store.BlockedUser, 0, len(flags))
+	for _, v := range flags {
+		blockedUsers = append(blockedUsers, v.(store.BlockedUser))
+	}
+	require.Equal(t, 1, len(blockedUsers))
+	blockedUserInfo := blockedUsers[0]
+	assert.Equal(t, "u1", blockedUserInfo.ID)
+	assert.True(t, blockedUserInfo.Until.After(time.Now().Add(time.Minute*59)), "blocked duration is more than 59m away")
+	assert.True(t, blockedUserInfo.Until.Before(time.Now().Add(time.Minute*61)), "blocked duration is less than 61m away")
 }
 
 func TestRPC_userDetailHndl(t *testing.T) {
@@ -301,6 +338,6 @@ func TestRPC_closeHndl(t *testing.T) {
 	api := fmt.Sprintf("http://localhost:%d/test", port)
 
 	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := re.Close()
-	assert.NoError(t, err)
+	assert.NoError(t, re.Close())
+	assert.NoError(t, re.Close(), "second call should not result in panic or errors")
 }

backend/_example/memory_store/server/image.go

@@ -35,7 +35,6 @@ func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.R
 	}
 	err := s.img.ResetCleanupTimer(fileID)
 	return jrpc.EncodeResponse(id, nil, err)
-
 }
 
 func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
@@ -47,6 +46,16 @@ func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response)
 	return jrpc.EncodeResponse(id, value, err)
 }
 
+func (s *RPC) imgDeleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Delete(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+
+}
+
 func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
 	var fileID string
 	if err := json.Unmarshal(params, &fileID); err != nil {

backend/_example/memory_store/server/image_test.go

@@ -158,4 +158,9 @@ func TestRPC_imgInfoHndl(t *testing.T) {
 	info, err = ri.Info()
 	assert.NoError(t, err)
 	assert.False(t, info.FirstStagingImageTS.IsZero())
+
+	err = ri.Delete("test_img")
+	assert.NoError(t, err)
+	_, err = ri.Load("test_img")
+	assert.EqualError(t, err, "image test_img not found")
 }

backend/_example/memory_store/server/rpc.go

@@ -60,6 +60,7 @@ func (s *RPC) addHandlers() {
 		"save_with_id":        s.imgSaveWithIDHndl,
 		"reset_cleanup_timer": s.imgResetClnTimerHndl,
 		"load":                s.imgLoadHndl,
+		"delete":              s.imgDeleteHndl,
 		"commit":              s.imgCommitHndl,
 		"cleanup":             s.imgCleanupHndl,
 		"info":                s.imgInfoHndl,

backend/_example/memory_store/server/rpc_test.go

@@ -47,7 +47,7 @@ func prepTestStore(t *testing.T) (port int, teardown func()) {
 	mg := accessor.NewMemData()
 	adm := accessor.NewMemAdminStore("secret")
 	img := accessor.NewMemImageStore()
-	s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+	s := NewRPC(mg, adm, img, jrpc.NewServer("/test"))
 
 	admRec := accessor.AdminRec{
 		SiteID:  "test-site",

backend/app/cmd/backup.go

@@ -14,11 +14,10 @@ import (
 // BackupCommand set of flags and command for export
 // ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
 type BackupCommand struct {
-	ExportPath  string        `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
-	ExportFile  string        `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"60m" description:"export (backup) timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
+	ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
+
+	SupportCmdOpts
 	CommonOpts
 }
 

backend/app/cmd/backup_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +18,10 @@ func TestBackup_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
 		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		fmt.Fprint(w, "blah\nblah2\n12345678\n")
 	}))
 	defer ts.Close()
@@ -34,6 +40,28 @@ func TestBackup_Execute(t *testing.T) {
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
+func TestBackup_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		require.Equal(t, "admin:", string(auth))
+		w.WriteHeader(http.StatusUnauthorized)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestBackup_ExecuteFailedStatus(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")

backend/app/cmd/cleanup.go

@@ -15,14 +15,14 @@ import (
 
 // CleanupCommand set of flags and command for cleanup
 type CleanupCommand struct {
-	Site        string   `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Dry         bool     `long:"dry" description:"dry mode, will not remove comments"`
-	From        string   `long:"from" description:"from yyyymmdd"`
-	To          string   `long:"to" description:"from yyyymmdd"`
-	BadWords    []string `short:"w" long:"bword" description:"bad word(s)"`
-	BadUsers    []string `short:"u" long:"buser" description:"bad user(s)"`
-	AdminPasswd string   `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
-	SetTitle    bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+	Dry      bool     `long:"dry" description:"dry mode, will not remove comments"`
+	From     string   `long:"from" description:"from yyyymmdd"`
+	To       string   `long:"to" description:"from yyyymmdd"`
+	BadWords []string `short:"w" long:"bword" description:"bad word(s)"`
+	BadUsers []string `short:"u" long:"buser" description:"bad user(s)"`
+	SetTitle bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+
+	SupportCmdOpts
 	CommonOpts
 }
 

backend/app/cmd/cleanup_test.go

@@ -195,7 +195,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
 	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
@@ -203,7 +203,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		c.lock.Unlock()
 	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()

backend/app/cmd/cmd.go

@@ -31,6 +31,14 @@ type CommonOpts struct {
 	Revision     string
 }
 
+// SupportCmdOpts is set of commands shared among similar commands like backup/restore and such.
+// Order of fields defines the help command output order.
+type SupportCmdOpts struct {
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	Timeout     time.Duration `long:"timeout" default:"60m" description:"timeout for the command run"`
+}
+
 // DeprecatedFlag contains information about deprecated option
 type DeprecatedFlag struct {
 	Old       string
@@ -107,6 +115,9 @@ func responseError(resp *http.Response) error {
 	if e != nil {
 		body = []byte("")
 	}
+	if resp.StatusCode == http.StatusUnauthorized {
+		return fmt.Errorf("error response %q, ensure you have set ADMIN_PASSWD and provided it to the command you're running: %s", resp.Status, body)
+	}
 	return fmt.Errorf("error response %q, %s", resp.Status, body)
 }
 

backend/app/cmd/import.go

@@ -8,18 +8,16 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
 
 	log "github.com/go-pkgz/lgr"
 )
 
 // ImportCommand set of flags and command for import
 type ImportCommand struct {
-	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"60m" description:"import timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
+	Provider  string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint
+
+	SupportCmdOpts
 	CommonOpts
 }
 

backend/app/cmd/import_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -18,6 +20,10 @@ func TestImport_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
@@ -46,6 +52,42 @@ func TestImport_Execute(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestImport_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestImport_ExecuteFailed(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")

backend/app/cmd/remap.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"time"
 
 	log "github.com/go-pkgz/lgr"
 )
@@ -14,10 +13,9 @@ import (
 // RemapCommand set of flags and command for change linkage between comments to
 // different urls based on given rules (input file)
 type RemapCommand struct {
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
-	Timeout     time.Duration `long:"timeout" default:"60m" description:"remap timeout"`
+	InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
+
+	SupportCmdOpts
 	CommonOpts
 }
 

backend/app/cmd/remap_test.go

@@ -1,9 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +19,10 @@ func TestRemap_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
@@ -33,3 +40,31 @@ func TestRemap_Execute(t *testing.T) {
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
+
+func TestRemap_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}

backend/app/cmd/restore.go

@@ -11,9 +11,7 @@ type RestoreCommand struct {
 	ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
 	ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
 
-	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
-	Timeout     time.Duration `long:"timeout" default:"60m" description:"import timeout"`
-	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	SupportCmdOpts
 	CommonOpts
 }
 
@@ -29,12 +27,10 @@ func (rc *RestoreCommand) Execute(args []string) error {
 		return err
 	}
 	importer := ImportCommand{
-		InputFile:   fname,
-		Site:        rc.Site,
-		Provider:    "native",
-		Timeout:     rc.Timeout,
-		AdminPasswd: rc.AdminPasswd,
-		CommonOpts:  rc.CommonOpts,
+		InputFile:      fname,
+		Provider:       "native",
+		SupportCmdOpts: rc.SupportCmdOpts,
+		CommonOpts:     rc.CommonOpts,
 	}
 	return importer.Execute(args)
 }

backend/app/cmd/server.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"context"
+	"embed"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -14,7 +16,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/jrpc"
-	"github.com/go-pkgz/lcw/eventbus"
+	"github.com/go-pkgz/lcw/v2/eventbus"
 	log "github.com/go-pkgz/lgr"
 	ntf "github.com/go-pkgz/notify"
 	"github.com/golang-jwt/jwt"
@@ -26,7 +28,7 @@ import (
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 
 	"github.com/umputun/remark42/backend/app/migrator"
 	"github.com/umputun/remark42/backend/app/notify"
@@ -41,6 +43,9 @@ import (
 	"github.com/umputun/remark42/backend/app/templates"
 )
 
+//go:embed web
+var webFS embed.FS
+
 // ServerCommand with command line flags and env
 type ServerCommand struct {
 	Store      StoreGroup      `group:"store" namespace:"store" env-namespace:"STORE"`
@@ -54,34 +59,36 @@ type ServerCommand struct {
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
-	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
-	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
-	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
-	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	AdminEdit        bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
-	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	Address          string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
-	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
-	RestrictedNames  []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
-	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
-	SimpleView       bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
-	ProxyCORS        bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
-	AllowedHosts     []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments"`
-	SubscribersOnly  bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
-	DisableSignature bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	Sites                      []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AnonymousVote              bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AdminPasswd                string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation             string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles             int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	LegacyImageProxy           bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	MinCommentSize             int           `long:"min-comment" env:"MIN_COMMENT_SIZE" default:"0" description:"min comment size"`
+	MaxCommentSize             int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes                   int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	RestrictVoteIP             bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	DurationVoteIP             time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	LowScore                   int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore              int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore              bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge                int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration               time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	AdminEdit                  bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
+	Port                       int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	Address                    string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
+	WebRoot                    string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit                float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords            []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedNames            []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
+	EnableEmoji                bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView                 bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
+	ProxyCORS                  bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
+	AllowedHosts               []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
+	SubscribersOnly            bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
+	DisableSignature           bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	DisableFancyTextFormatting bool          `long:"disable-fancy-text-formatting" env:"DISABLE_FANCY_TEXT_FORMATTING" description:"disable fancy comments text formatting (replacement of quotes, dashes, fractions, etc)"`
 
 	Auth struct {
 		TTL struct {
@@ -92,16 +99,17 @@ type ServerCommand struct {
 		SendJWTHeader bool   `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"send JWT as a header instead of cookie"`
 		SameSite      string `long:"same-site" env:"SAME_SITE" description:"set same site policy for cookies" choice:"default" choice:"none" choice:"lax" choice:"strict" default:"default"` // nolint
 
-		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Microsoft AuthGroup `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
-		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
-		Patreon   AuthGroup `group:"patreon" namespace:"patreon" env-namespace:"PATREON" description:"Patreon OAuth"`
-		Telegram  bool      `long:"telegram" env:"TELEGRAM" description:"Enable Telegram auth (using token from telegram.token)"`
-		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
-		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Apple     AppleGroup `group:"apple" namespace:"apple" env-namespace:"APPLE" description:"Apple OAuth"`
+		Google    AuthGroup  `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Github    AuthGroup  `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Facebook  AuthGroup  `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Microsoft AuthGroup  `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
+		Yandex    AuthGroup  `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Twitter   AuthGroup  `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
+		Patreon   AuthGroup  `group:"patreon" namespace:"patreon" env-namespace:"PATREON" description:"Patreon OAuth"`
+		Telegram  bool       `long:"telegram" env:"TELEGRAM" description:"Enable Telegram auth (using token from telegram.token)"`
+		Dev       bool       `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool       `long:"anon" env:"ANON" description:"enable anonymous login"`
 		Email     struct {
 			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
 			From         string        `long:"from" env:"FROM" description:"from email address"`
@@ -129,6 +137,14 @@ type ImageProxyGroup struct {
 	CacheExternal bool `long:"cache-external" env:"CACHE_EXTERNAL" description:"enable caching for external images"`
 }
 
+// AppleGroup defines options for Apple auth params
+type AppleGroup struct {
+	CID                string `long:"cid" env:"CID" description:"Apple client ID"`
+	TID                string `long:"tid" env:"TID" description:"Apple service ID"`
+	KID                string `long:"kid" env:"KID" description:"Private key ID"`
+	PrivateKeyFilePath string `long:"private-key-filepath" env:"PRIVATE_KEY_FILEPATH" description:"Private key file location" default:"/srv/var/apple.p8"`
+}
+
 // AuthGroup defines options group for auth params
 type AuthGroup struct {
 	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
@@ -193,7 +209,7 @@ type AdminGroup struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  []string `long:"email" env:"EMAIL" description:"admin emails" env-delim:","`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
-	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
+	RPC AdminRPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // TelegramGroup defines token for Telegram used in notify and auth modules
@@ -204,13 +220,15 @@ type TelegramGroup struct {
 
 // SMTPGroup defines options for SMTP server connection, used in auth and notify modules
 type SMTPGroup struct {
-	Host     string        `long:"host" env:"HOST" description:"SMTP host"`
-	Port     int           `long:"port" env:"PORT" description:"SMTP port"`
-	Username string        `long:"username" env:"USERNAME" description:"SMTP user name"`
-	Password string        `long:"password" env:"PASSWORD" description:"SMTP password"`
-	TLS      bool          `long:"tls" env:"TLS" description:"enable TLS"`
-	StartTLS bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
-	TimeOut  time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
+	Host               string        `long:"host" env:"HOST" description:"SMTP host"`
+	Port               int           `long:"port" env:"PORT" description:"SMTP port"`
+	Username           string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Password           string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	TLS                bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	InsecureSkipVerify bool          `long:"insecure_skip_verify" env:"INSECURE_SKIP_VERIFY" description:"skip certificate verification"`
+	LoginAuth          bool          `long:"login_auth" env:"LOGIN_AUTH" description:"enable LOGIN auth instead of PLAIN"`
+	StartTLS           bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
+	TimeOut            time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
 }
 
 // NotifyGroup defines options for notification
@@ -260,6 +278,12 @@ type RPCGroup struct {
 	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
 }
 
+// AdminRPCGroup defines options for remote admin store
+type AdminRPCGroup struct {
+	RPCGroup
+	SecretPerSite bool `long:"secret_per_site" env:"SECRET_PER_SITE" description:"enable JWT secret retrieval per aud, which is site_id in this case"`
+}
+
 // LoadingCache defines interface for caching
 type LoadingCache interface {
 	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
@@ -480,11 +504,12 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		EditDuration:           s.EditDuration,
 		AdminEdits:             s.AdminEdit,
 		AdminStore:             adminStore,
+		MinCommentSize:         s.MinCommentSize,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
 		PositiveScore:          s.PositiveScore,
 		ImageService:           imageService,
-		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}, s.getAllowedDomains()),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
 	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
@@ -505,11 +530,12 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	authenticator := s.getAuthenticator(dataService, avatarStore, adminStore, authRefreshCache)
 
 	telegramAuth := s.makeTelegramAuth(authenticator) // telegram auth requires TelegramAPI listener which is constructed below
-	telegramService, telegramBotUsername := s.startTelegramAuthAndNotify(ctx, telegramAuth)
+	telegramService := s.startTelegramAuthAndNotify(ctx, telegramAuth)
 
 	err = s.addAuthProviders(authenticator)
 	if err != nil {
 		_ = dataService.Close()
+		_ = authRefreshCache.Close()
 		return nil, fmt.Errorf("failed to make authenticator: %w", err)
 	}
 
@@ -519,7 +545,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		Cache:             loadingCache,
 		NativeImporter:    &migrator.Native{DataStore: dataService},
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
-		WordPressImporter: &migrator.WordPress{DataStore: dataService},
+		WordPressImporter: &migrator.WordPress{DataStore: dataService, DisableFancyTextFormatting: s.DisableFancyTextFormatting},
 		CommentoImporter:  &migrator.Commento{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
 		URLMapperMaker:    migrator.NewURLMapper,
@@ -549,36 +575,39 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
 		_ = dataService.Close()
+		_ = authRefreshCache.Close()
 		return nil, fmt.Errorf("failed to make config of ssl server params: %w", err)
 	}
 
 	srv := &api.Rest{
-		Version:             s.Revision,
-		DataService:         dataService,
-		WebRoot:             s.WebRoot,
-		RemarkURL:           s.RemarkURL,
-		ImageProxy:          imgProxy,
-		CommentFormatter:    commentFormatter,
-		Migrator:            migr,
-		ReadOnlyAge:         s.ReadOnlyAge,
-		SharedSecret:        s.SharedSecret,
-		Authenticator:       authenticator,
-		Cache:               loadingCache,
-		NotifyService:       notifyService,
-		TelegramService:     telegramService,
-		SSLConfig:           sslConfig,
-		UpdateLimiter:       s.UpdateLimit,
-		ImageService:        imageService,
-		EmailNotifications:  contains("email", s.Notify.Users),
-		TelegramBotUsername: telegramBotUsername,
-		EmojiEnabled:        s.EnableEmoji,
-		AnonVote:            s.AnonymousVote && s.RestrictVoteIP,
-		SimpleView:          s.SimpleView,
-		ProxyCORS:           s.ProxyCORS,
-		AllowedAncestors:    s.AllowedHosts,
-		SendJWTHeader:       s.Auth.SendJWTHeader,
-		SubscribersOnly:     s.SubscribersOnly,
-		DisableSignature:    s.DisableSignature,
+		Version:                    s.Revision,
+		DataService:                dataService,
+		WebRoot:                    s.WebRoot,
+		WebFS:                      webFS,
+		RemarkURL:                  s.RemarkURL,
+		ImageProxy:                 imgProxy,
+		CommentFormatter:           commentFormatter,
+		Migrator:                   migr,
+		ReadOnlyAge:                s.ReadOnlyAge,
+		SharedSecret:               s.SharedSecret,
+		Authenticator:              authenticator,
+		Cache:                      loadingCache,
+		NotifyService:              notifyService,
+		TelegramService:            telegramService,
+		SSLConfig:                  sslConfig,
+		UpdateLimiter:              s.UpdateLimit,
+		ImageService:               imageService,
+		EmailNotifications:         contains("email", s.Notify.Users),
+		TelegramNotifications:      contains("telegram", s.Notify.Users) && telegramService != nil,
+		EmojiEnabled:               s.EnableEmoji,
+		AnonVote:                   s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:                 s.SimpleView,
+		ProxyCORS:                  s.ProxyCORS,
+		AllowedAncestors:           s.AllowedHosts,
+		SendJWTHeader:              s.Auth.SendJWTHeader,
+		SubscribersOnly:            s.SubscribersOnly,
+		DisableSignature:           s.DisableSignature,
+		DisableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -588,6 +617,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		da, errDevAuth := authenticator.DevAuth()
 		if errDevAuth != nil {
 			_ = dataService.Close()
+			_ = authRefreshCache.Close()
 			return nil, fmt.Errorf("can't make dev oauth2 server: %w", errDevAuth)
 		}
 		devAuth = da
@@ -609,6 +639,47 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	}, nil
 }
 
+// Extract domains from s.AllowedHosts and second level domain from s.RemarkURL.
+// It can be and IP like http://127.0.0.1 in which case we need to use whole IP as domain
+// Beware, if s.RemarkURL is in third-level domain like https://example.co.uk, co.uk will be returned.
+func (s *ServerCommand) getAllowedDomains() []string {
+	rawDomains := s.AllowedHosts
+	rawDomains = append(rawDomains, s.RemarkURL)
+	allowedDomains := []string{}
+	for _, rawURL := range rawDomains {
+		// case of 'self' AllowedHosts, which is not a valid rawURL name
+		if rawURL == "self" || rawURL == "'self'" || rawURL == "\"self\"" {
+			continue
+		}
+		// AllowedHosts usually don't have https:// prefix, so we're adding it just to make parsing below work the same way as for RemarkURL
+		if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
+			rawURL = "https://" + rawURL
+		}
+		parsedURL, err := url.Parse(rawURL)
+		if err != nil {
+			log.Printf("[WARN] failed to parse URL %s for TitleExtract whitelist: %v", rawURL, err)
+			continue
+		}
+		domain := parsedURL.Hostname()
+
+		if domain == "" || // don't add empty domain as it will allow everything to be extracted
+			(len(strings.Split(domain, ".")) < 2 && // don't allow single-word domains like "com"
+				domain != "localhost") { // localhost is an exceptional single-word domain which is allowed
+			continue
+		}
+
+		// Only for RemarkURL if domain is not IP and has more than two levels, extract second level domain.
+		// For AllowedHosts we don't do this as they are exact list of domains which can host comments, but
+		// RemarkURL might be on a subdomain and we must allow parent domain to be used for TitleExtract.
+		if rawURL == s.RemarkURL && net.ParseIP(domain) == nil && len(strings.Split(domain, ".")) > 2 {
+			domain = strings.Join(strings.Split(domain, ".")[len(strings.Split(domain, "."))-2:], ".")
+		}
+
+		allowedDomains = append(allowedDomains, domain)
+	}
+	return allowedDomains
+}
+
 // Run all application objects
 func (a *serverApp) run(ctx context.Context) error {
 	if a.AdminPasswd != "" {
@@ -798,37 +869,54 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 
 func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
+	o := cache.NewOpts[[]byte]()
 	switch s.Cache.Type {
 	case "redis_pub_sub":
 		redisPubSub, err := eventbus.NewRedisPubSub(s.Cache.RedisAddr, "remark42-cache")
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization, redis PubSub initialisation: %w", err)
 		}
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items), cache.EventBus(redisPubSub))
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
+			o.MaxKeys(s.Cache.Max.Items), o.EventBus(redisPubSub))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "mem":
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items))
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
+			o.MaxKeys(s.Cache.Max.Items))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "none":
-		return cache.NewScache(&cache.Nop{}), nil
+		return cache.NewScache[[]byte](&cache.Nop[[]byte]{}), nil
 	}
 	return nil, fmt.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
+//nolint:gocyclo // simple code but many if checks
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 	providersCount := 0
 	if s.Auth.Telegram {
 		providersCount++
 	}
 
+	if s.Auth.Apple.CID != "" && s.Auth.Apple.TID != "" && s.Auth.Apple.KID != "" {
+		err := authenticator.AddAppleProvider(
+			provider.AppleConfig{
+				ClientID:     s.Auth.Apple.CID,
+				TeamID:       s.Auth.Apple.TID,
+				KeyID:        s.Auth.Apple.KID,
+				ResponseMode: "query", // default is form_post which wouldn't work here
+			},
+			provider.LoadApplePrivateKeyFromFile(s.Auth.Apple.PrivateKeyFilePath),
+		)
+		if err != nil {
+			return err
+		}
+		providersCount++
+	}
 	if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
 		authenticator.AddProvider("google", s.Auth.Google.CID, s.Auth.Google.CSEC)
 		providersCount++
@@ -860,28 +948,36 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 
 	if s.Auth.Dev {
 		log.Print("[INFO] dev access enabled")
-		authenticator.AddProvider("dev", "", "")
+		u, errURL := url.Parse(s.RemarkURL)
+		if errURL != nil {
+			return fmt.Errorf("can't parse Remark42 URL: %w", errURL)
+		}
+		authenticator.AddDevProvider(u.Hostname(), 8084)
 		providersCount++
 	}
 
 	if s.Auth.Email.Enable {
 		params := sender.EmailParams{
-			Host:         s.SMTP.Host,
-			Port:         s.SMTP.Port,
-			SMTPUserName: s.SMTP.Username,
-			SMTPPassword: s.SMTP.Password,
-			TimeOut:      s.SMTP.TimeOut,
-			TLS:          s.SMTP.TLS,
-			From:         s.Auth.Email.From,
-			Subject:      s.Auth.Email.Subject,
-			ContentType:  s.Auth.Email.ContentType,
+			Host:               s.SMTP.Host,
+			Port:               s.SMTP.Port,
+			SMTPUserName:       s.SMTP.Username,
+			SMTPPassword:       s.SMTP.Password,
+			TimeOut:            s.SMTP.TimeOut,
+			StartTLS:           s.SMTP.StartTLS,
+			LoginAuth:          s.SMTP.LoginAuth,
+			TLS:                s.SMTP.TLS,
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
+			Charset:            "UTF-8",
+			From:               s.Auth.Email.From,
+			Subject:            s.Auth.Email.Subject,
+			ContentType:        s.Auth.Email.ContentType,
 		}
 		sndr := sender.NewEmailClient(params, log.Default())
-		tmpl, err := s.loadEmailTemplate()
+		tmpl, err := templates.Read(s.Auth.Email.MsgTemplate)
 		if err != nil {
 			return err
 		}
-		authenticator.AddVerifProvider("email", tmpl, sndr)
+		authenticator.AddVerifProvider("email", string(tmpl), sndr)
 	}
 
 	if s.Auth.Anonymous {
@@ -925,27 +1021,6 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 	return nil
 }
 
-// loadEmailTemplate trying to get template from statik
-func (s *ServerCommand) loadEmailTemplate() (string, error) {
-	var file []byte
-	var err error
-
-	if s.Auth.Email.MsgTemplate == "email_confirmation_login.html.tmpl" {
-		fs := templates.NewFS()
-		file, err = fs.ReadFile(s.Auth.Email.MsgTemplate)
-	} else {
-		// deprecated loading from an external file, should be removed before v1.9.0
-		file, err = os.ReadFile(s.Auth.Email.MsgTemplate)
-		log.Printf("[INFO] template %s will be read from disk", s.Auth.Email.MsgTemplate)
-	}
-
-	if err != nil {
-		return "", fmt.Errorf("failed to read file %s: %w", s.Auth.Email.MsgTemplate, err)
-	}
-
-	return string(file), nil
-}
-
 // creates and registers telegram auth, which we need separately from other auth providers
 func (s *ServerCommand) makeTelegramAuth(authenticator *auth.Service) providers.TGUpdatesReceiver {
 	if s.Auth.Telegram {
@@ -967,7 +1042,8 @@ func (s *ServerCommand) makeNotifyService(dataStore *service.DataStore, destinat
 	if destinations == nil {
 		destinations = []notify.Destination{}
 	}
-	if telegram != nil {
+	// it's possible that telegram notification service was created for auth but should not be used for notifications
+	if telegram != nil && (contains("telegram", s.Notify.Users) || contains("telegram", s.Notify.Admins)) {
 		destinations = append(destinations, telegram)
 	}
 
@@ -1038,15 +1114,17 @@ func (s *ServerCommand) makeNotifyDestinations(authenticator *auth.Service) ([]n
 			emailParams.AdminEmails = s.Admin.Shared.Email
 		}
 		smtpParams := ntf.SMTPParams{
-			Host:        s.SMTP.Host,
-			Port:        s.SMTP.Port,
-			TLS:         s.SMTP.TLS,
-			StartTLS:    s.SMTP.StartTLS,
-			Username:    s.SMTP.Username,
-			Password:    s.SMTP.Password,
-			TimeOut:     s.SMTP.TimeOut,
-			ContentType: "text/html",
-			Charset:     "UTF-8",
+			Host:               s.SMTP.Host,
+			Port:               s.SMTP.Port,
+			TLS:                s.SMTP.TLS,
+			StartTLS:           s.SMTP.StartTLS,
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
+			LoginAuth:          s.SMTP.LoginAuth,
+			Username:           s.SMTP.Username,
+			Password:           s.SMTP.Password,
+			TimeOut:            s.SMTP.TimeOut,
+			ContentType:        "text/html",
+			Charset:            "UTF-8",
 		}
 		emailService, err := notify.NewEmail(emailParams, smtpParams)
 		if err != nil {
@@ -1118,7 +1196,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 		SameSiteCookie: s.parseSameSite(s.Auth.SameSite),
 		SecureCookies:  strings.HasPrefix(s.RemarkURL, "https://"),
 		SecretReader: token.SecretFunc(func(aud string) (string, error) { // get secret per site
-			return admns.Key("")
+			return admns.Key(aud)
 		}),
 		ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
 			if c.User == nil {
@@ -1147,7 +1225,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
-		Validator: token.ValidatorFunc(func(token string, claims token.Claims) bool { // check on each auth call (in middleware)
+		Validator: token.ValidatorFunc(func(_ string, claims token.Claims) bool { // check on each auth call (in middleware)
 			if claims.User == nil {
 				return false
 			}
@@ -1163,6 +1241,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 		Logger:            log.Default(),
 		RefreshCache:      authRefreshCache,
 		UseGravatar:       true,
+		AudSecrets:        s.Admin.RPC.SecretPerSite,
 	})
 }
 
@@ -1183,20 +1262,15 @@ func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
 
 // startTelegramAuthAndNotify initializes telegram notify and auth Telegram Bot listen loop.
 // Does nothing if telegram auth and notifications are disabled.
-// Doesn't return telegram bot username if user notifications are disabled, as that is the way frontend knows they are enabled.
-func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram, telegramBotUsername string) {
+func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram) {
 	if !contains("telegram", s.Notify.Users) && !contains("telegram", s.Notify.Admins) && !s.Auth.Telegram {
-		return nil, ""
+		return nil
 	}
 
 	var err error
 	if tg, err = s.makeTelegramNotify(); err != nil {
 		log.Printf("[WARN] failed to make telegram notify service, %s", err)
-		return nil, ""
-	}
-
-	if contains("telegram", s.Notify.Users) {
-		telegramBotUsername = tg.GetBotUsername()
+		return nil
 	}
 
 	telegramReceivers := []providers.TGUpdatesReceiver{tg}
@@ -1206,7 +1280,7 @@ func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegram
 	// start bot messages receiver for both notify and auth services
 	go providers.DispatchTelegramUpdates(ctx, tg, telegramReceivers, time.Second*5)
 
-	return tg, telegramBotUsername
+	return tg
 }
 
 // splitAtCommas split s at commas, ignoring commas in strings.
@@ -1253,11 +1327,12 @@ func splitAtCommas(s string) []string {
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	cache.LoadingCache
+	cache.LoadingCache[token.Claims]
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	expirableCache, _ := cache.NewExpirableCache(cache.TTL(5 * time.Minute))
+	o := cache.NewOpts[token.Claims]()
+	expirableCache, _ := cache.NewExpirableCache(o.TTL(5 * time.Minute))
 	return &authRefreshCache{LoadingCache: expirableCache}
 }
 
@@ -1268,5 +1343,5 @@ func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 
 // Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	_, _ = c.LoadingCache.Get(key.(string), func() (interface{}, error) { return value, nil })
+	_, _ = c.LoadingCache.Get(key.(string), func() (token.Claims, error) { return value.(token.Claims), nil })
 }

backend/app/cmd/server_test.go

@@ -79,7 +79,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	waitForHTTPServerStart(port)
 
 	providers := app.restSrv.Authenticator.Providers()
-	require.Equal(t, 9+1, len(providers), "extra auth provider")
+	require.Equal(t, 10+1, len(providers), "extra auth provider")
 	assert.Equal(t, "dev", providers[len(providers)-2].Name(), "dev auth provider")
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
@@ -107,7 +107,7 @@ func TestServerApp_AnonMode(t *testing.T) {
 	waitForHTTPServerStart(port)
 
 	providers := app.restSrv.Authenticator.Providers()
-	require.Equal(t, 9+1, len(providers), "extra auth provider for anon")
+	require.Equal(t, 10+1, len(providers), "extra auth provider for anon")
 	assert.Equal(t, "anonymous", providers[len(providers)-1].Name(), "anon auth provider")
 
 	client := http.Client{Timeout: 10 * time.Second}
@@ -143,36 +143,42 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 
 	// try to login with non-latin name
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	// try to login with bad name
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with short name
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with name what have space in prefix
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with name what have space in suffix
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with long name
+	time.Sleep(time.Second)
 	ln := strings.Repeat("x", 65)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
 	require.NoError(t, err)
@@ -180,12 +186,14 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// try to login with admin name
+	time.Sleep(time.Second)
 	resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	// try to add a comment as anonymous with admin name
+	time.Sleep(time.Second)
 	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	require.NoError(t, err)
@@ -242,7 +250,7 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -282,7 +290,8 @@ func TestServerApp_WithRemote(t *testing.T) {
 	port := chooseRandomUnusedPort()
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
 		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
-		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+		"--port=" + strconv.Itoa(port), "--avatar.fs.path=/tmp",
+		"--admin.type=rpc", "--admin.rpc.secret_per_site", "--admin.rpc.api=http://127.0.0.1"})
 	require.NoError(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
 	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
@@ -343,6 +352,7 @@ func TestServerApp_Failed(t *testing.T) {
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
 
+	// wrong store type
 	opts = ServerCommand{}
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
@@ -366,6 +376,19 @@ func TestServerApp_Failed(t *testing.T) {
 			"problem subscribing to channel remark42-cache on address wrong_address: "+
 			"dial tcp: address wrong_address: missing port in address")
 	t.Log(err)
+
+	// wrong apple private key type
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+	p = flags.NewParser(&opts, flags.Default)
+	_, err = p.ParseArgs([]string{"--auth.apple.cid=123", "--auth.apple.tid=123",
+		"--auth.apple.kid=123", "--auth.apple.private-key-filepath=testdata/apple-bad.p8"})
+	assert.NoError(t, err)
+	_, err = opts.newServerApp(context.Background())
+	assert.EqualError(t, err,
+		"failed to make authenticator: an AppleProvider creating failed: "+
+			"provided private key is not ECDSA")
+	t.Log(err)
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
@@ -589,7 +612,7 @@ func TestServerAuthHooks(t *testing.T) {
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID:   "dev",
+			ID:   "github_dev",
 			Name: "developer one",
 		},
 	}
@@ -628,14 +651,14 @@ func TestServerAuthHooks(t *testing.T) {
 	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
 
-	// block user dev as admin
+	// block user github_dev as admin
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), http.NoBody)
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/github_dev?site=remark&block=1&ttl=10d", port), http.NoBody)
 	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "user github_dev blocked")
 	b, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
@@ -659,19 +682,6 @@ func TestServerAuthHooks(t *testing.T) {
 	client.CloseIdleConnections()
 }
 
-func TestServer_loadEmailTemplate(t *testing.T) {
-	cmd := ServerCommand{}
-	cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
-	r, err := cmd.loadEmailTemplate()
-	assert.NoError(t, err)
-	assert.Equal(t, "The token is {{.Token}}", r)
-
-	cmd.Auth.Email.MsgTemplate = "badpath.tmpl"
-	r, err = cmd.loadEmailTemplate()
-	assert.EqualError(t, err, "failed to read file badpath.tmpl: open badpath.tmpl: no such file or directory")
-	assert.Equal(t, r, "")
-}
-
 func TestServerCommand_parseSameSite(t *testing.T) {
 	tbl := []struct {
 		inp string
@@ -716,6 +726,28 @@ func Test_splitAtCommas(t *testing.T) {
 	}
 }
 
+func Test_getAllowedDomains(t *testing.T) {
+	tbl := []struct {
+		s              ServerCommand
+		allowedDomains []string
+	}{
+		// correct example, parsed and returned as allowed domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "https://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://localhost"}}, []string{"localhost"}},
+		// incorrect URLs, so Hostname is empty but returned list doesn't include empty string as it would allow any domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "bad hostname"}}, []string{}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "not_a_hostname"}}, []string{}},
+		// test removal of 'self', multiple AllowedHosts. No deduplication is expected
+		{ServerCommand{AllowedHosts: []string{"'self'", "example.org", "test.example.org", "remark42.com"}, CommonOpts: CommonOpts{RemarkURL: "https://example.org"}}, []string{"example.org", "test.example.org", "remark42.com", "example.org"}},
+	}
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.allowedDomains, tt.s.getAllowedDomains())
+		})
+	}
+}
+
 func chooseRandomUnusedPort() (port int) {
 	for i := 0; i < 10; i++ {
 		port = 40000 + int(rand.Int31n(10000))
@@ -763,6 +795,8 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
 	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp/remark42_test", "fs", "/tmp/remark42_test", "/tmp/remark42_test"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
+	cmd.Auth.Apple.CID, cmd.Auth.Apple.KID, cmd.Auth.Apple.TID = "cid", "kid", "tid"
+	cmd.Auth.Apple.PrivateKeyFilePath = "testdata/apple.p8"
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
@@ -809,12 +843,15 @@ func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Cont
 	ctx, cancel := context.WithCancel(context.Background())
 	app, err := cmd.newServerApp(ctx)
 	require.NoError(t, err)
-
-	rand.Seed(time.Now().UnixNano())
 	return app, ctx, cancel
 }
 
 func TestMain(m *testing.M) {
 	// ignore is added only for GitHub Actions, can't reproduce locally
-	goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
+	goleak.VerifyTestMain(
+		m,
+		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/cmd/testdata/apple-bad.p8

@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKNwapOQ6rQJHetP
+HRlJBIh1OsOsUBiXb3rXXE3xpWAxAha0MH+UPRblOko+5T2JqIb+xKf9Vi3oTM3t
+KvffaOPtzKXZauscjq6NGzA3LgeiMy6q19pvkUUOlGYK6+Xfl+B7Xw6+hBMkQuGE
+nUS8nkpR5mK4ne7djIyfHFfMu4ptAgMBAAECgYA+s0PPtMq1osG9oi4xoxeAGikf
+JB3eMUptP+2DYW7mRibc+ueYKhB9lhcUoKhlQUhL8bUUFVZYakP8xD21thmQqnC4
+f63asad0ycteJMLb3r+z26LHuCyOdPg1pyLk3oQ32lVQHBCYathRMcVznxOG16VK
+I8BFfstJTaJu0lK/wQJBANYFGusBiZsJQ3utrQMVPpKmloO2++4q1v6ZR4puDQHx
+TjLjAIgrkYfwTJBLBRZxec0E7TmuVQ9uJ+wMu/+7zaUCQQDDf2xMnQqYknJoKGq+
+oAnyC66UqWC5xAnQS32mlnJ632JXA0pf9pb1SXAYExB1p9Dfqd3VAwQDwBsDDgP6
+HD8pAkEA0lscNQZC2TaGtKZk2hXkdcH1SKru/g3vWTkRHxfCAznJUaza1fx0wzdG
+GcES1Bdez0tbW4llI5By/skZc2eE3QJAFl6fOskBbGHde3Oce0F+wdZ6XIJhEgCP
+iukIcKZoZQzoiMJUoVRrA5gqnmaYDI5uRRl/y57zt6YksR3KcLUIuQJAd242M/WF
+6YAZat3q/wEeETeQq1wrooew+8lHl05/Nt0cCpV48RGEhJ83pzBm3mnwHf8lTBJH
+x6XroMXsmbnsEw==
+-----END PRIVATE KEY-----

backend/app/cmd/testdata/apple.p8

@@ -0,0 +1,6 @@
+-----BEGIN PRIVATE KEY-----
+MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgGH2MylyZjjRdauTk
+xxXW6p8VSHqIeVRRKSJPg1xn6+KgCgYIKoZIzj0DAQehRANCAAS/mNzQ7aBbIBr3
+DiHiJGIDEzi6+q3mmyhH6ZWQWFdFei2qgdyM1V6qtRPVq+yHBNSBebbR4noE/IYO
+hMdWYrKn
+-----END PRIVATE KEY-----

backend/app/cmd/web/index.html

@@ -0,0 +1 @@
+This stub page would be replaced by the frontend statically built HTML during the Docker image build.

backend/app/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -23,7 +24,8 @@ type Opts struct {
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
 	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
-	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	// SharedSecret is only used in server command, but defined for all commands for historical reasons
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"the shared secret key used to sign JWT, should be a random, long, hard-to-guess string"`
 
 	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
@@ -54,11 +56,11 @@ func main() {
 	}
 
 	if _, err := p.Parse(); err != nil {
-		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
+		var flagsErr *flags.Error
+		if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp {
 			os.Exit(0)
-		} else {
-			os.Exit(1)
 		}
+		os.Exit(1)
 	}
 }
 
@@ -97,7 +99,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
-// nolint:gochecknoinits // can't avoid it in this place
+//nolint:gochecknoinits // can't avoid it in this place
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal, 1)

backend/app/main_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -22,7 +21,7 @@ import (
 )
 
 func Test_Main(t *testing.T) {
-	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
+	dir, err := os.MkdirTemp(os.TempDir(), "remark42")
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
@@ -60,12 +59,12 @@ func Test_Main(t *testing.T) {
 }
 
 func TestMain_WithWebhook(t *testing.T) {
-	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
+	dir, err := os.MkdirTemp(os.TempDir(), "remark42")
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
 	var webhookSent int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
 		atomic.StoreInt32(&webhookSent, 1)
 		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
 
@@ -158,5 +157,7 @@ func TestMain(m *testing.M) {
 		m,
 		goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
 		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
 	)
 }

backend/app/migrator/backup.go

@@ -83,7 +83,7 @@ func (ab AutoBackup) removeOldBackupFiles() {
 			backFiles = append(backFiles, info)
 		}
 	}
-	sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
+	sort.Slice(backFiles, func(i, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
 
 	if len(backFiles) > ab.KeepMax {
 		for i := 0; i < len(backFiles)-ab.KeepMax; i++ {

backend/app/migrator/commento.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"time"
 
 	"github.com/umputun/remark42/backend/app/store"
@@ -110,16 +111,28 @@ func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				continue
 			}
 
+			parentID := comment.ParentHex
+			// comments with ParentHex == "root" are top-level comments
+			if parentID == "root" {
+				parentID = ""
+			}
+
+			commentURL, e := url.JoinPath("https://", comment.Domain, comment.Path)
+			if e != nil {
+				log.Printf("[WARN] can't construct comment URL in commento import, %s", err.Error())
+			}
+			log.Printf("[ERROR] commentoURL: %s", commentURL)
+
 			c := store.Comment{
 				ID: comment.CommentHex,
 				Locator: store.Locator{
-					URL:    comment.Path,
+					URL:    commentURL,
 					SiteID: siteID,
 				},
 				User:      u,
 				Text:      comment.Markdown,
 				Timestamp: comment.CreationDate,
-				ParentID:  comment.ParentHex,
+				ParentID:  parentID,
 				Imported:  true,
 			}
 

backend/app/migrator/migrator.go

@@ -38,7 +38,7 @@ type MapperMaker func(reader io.Reader) (Mapper, error)
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
 	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	List(siteID string, limit, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
 	SetMetas(siteID string, umetas []service.UserMetaData, pmetas []service.PostMetaData) error

backend/app/migrator/native_test.go

@@ -163,7 +163,7 @@ func TestNative_ImportManyWithError(t *testing.T) {
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
 	for i := 0; i < 100; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
+		fmt.Fprintf(buf, goodRec, i)
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")

backend/app/migrator/testdata/commento.json

@@ -4,7 +4,7 @@
 	{
 	  "commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "anonymous",
 	  "markdown": "Example comment created by user.",
 	  "html": "",
@@ -18,7 +18,7 @@
 	{
 	  "commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 	  "markdown": "Example 2 comment created by user.",
 	  "html": "",
@@ -32,7 +32,7 @@
 	{
 	  "commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
 	  "markdown": "Great reply!",
 	  "html": "",

backend/app/migrator/wordpress.go

@@ -16,7 +16,8 @@ const wpTimeLayout = "2006-01-02 15:04:05"
 
 // WordPress implements Importer from WP xml
 type WordPress struct {
-	DataStore Store
+	DataStore                  Store
+	DisableFancyTextFormatting bool
 }
 
 type wpItem struct {
@@ -138,7 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								ParentID:  comment.PID,
 								Imported:  true,
 							}
-							commentsCh <- commentFormatter.Format(c)
+							commentsCh <- commentFormatter.Format(c, w.DisableFancyTextFormatting)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
 								log.Printf("[DEBUG] processed %d comments", stats.inpComments)

backend/app/migrator/wordpress_test.go

@@ -24,7 +24,7 @@ func TestWordPress_Import(t *testing.T) {
 
 	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 	defer dataStore.Close()
-	wp := WordPress{DataStore: &dataStore}
+	wp := WordPress{DataStore: &dataStore, DisableFancyTextFormatting: false}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
@@ -41,7 +41,7 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
-	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.Equal(t, "<p>«Mekkatorque» was over in that tent up to the right</p>\n", c.Text)
 	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
@@ -54,6 +54,18 @@ func TestWordPress_Import(t *testing.T) {
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
+
+	// test with DisableFancyTextFormatting
+	wp = WordPress{DataStore: &dataStore, DisableFancyTextFormatting: true}
+	size, err = wp.Import(strings.NewReader(xmlTestWP), siteID)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, size)
+
+	last, err = dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
+
+	assert.Equal(t, "<p>&#34;Mekkatorque&#34; was over in that tent up to the right</p>\n", last[0].Text)
 }
 
 func TestWordPress_Convert(t *testing.T) {
@@ -247,7 +259,7 @@ var xmlTestWP = `
 			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
 			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
 			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
-			<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
+			<wp:comment_content><![CDATA["Mekkatorque" was over in that tent up to the right]]></wp:comment_content>
 			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
 			<wp:comment_type><![CDATA[]]></wp:comment_type>
 			<wp:comment_parent>13</wp:comment_parent>

backend/app/notify/email.go

@@ -100,7 +100,6 @@ func NewEmail(emailParams EmailParams, smtpParams ntf.SMTPParams) (*Email, error
 func (e *Email) setTemplates() error {
 	var err error
 	var msgTmplFile, verifyTmplFile []byte
-	fs := templates.NewFS()
 
 	if e.VerificationTemplatePath == "" {
 		e.VerificationTemplatePath = defaultEmailVerificationTemplatePath
@@ -110,10 +109,10 @@ func (e *Email) setTemplates() error {
 		e.MsgTemplatePath = defaultEmailTemplatePath
 	}
 
-	if msgTmplFile, err = fs.ReadFile(e.MsgTemplatePath); err != nil {
+	if msgTmplFile, err = templates.Read(e.MsgTemplatePath); err != nil {
 		return fmt.Errorf("can't read message template: %w", err)
 	}
-	if verifyTmplFile, err = fs.ReadFile(e.VerificationTemplatePath); err != nil {
+	if verifyTmplFile, err = templates.Read(e.VerificationTemplatePath); err != nil {
 		return fmt.Errorf("can't read verification template: %w", err)
 	}
 	if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil {

backend/app/notify/email_test.go

@@ -55,32 +55,18 @@ func Test_initTemplatesErr(t *testing.T) {
 		errText     string
 		emailParams EmailParams
 	}{
-		{
-			name:        "with wrong (default, working in prod) path to reply template",
-			errText:     "can't read message template: open email_reply.html.tmpl: no such file or directory",
-			emailParams: EmailParams{},
-		},
-		{
-			name:    "with wrong (default, working in prod) path to verification template",
-			errText: "can't read verification template: open email_confirmation_subscription.html.tmpl: no such file or directory",
-			emailParams: EmailParams{
-				MsgTemplatePath: "testdata/msg.html.tmpl",
-			},
-		},
 		{
 			name:    "with wrong path to verification template",
-			errText: "can't read verification template: open notfound.tmpl: no such file or directory",
+			errText: "notfound.tmpl: file does not exist",
 			emailParams: EmailParams{
 				VerificationTemplatePath: "notfound.tmpl",
-				MsgTemplatePath:          "testdata/msg.html.tmpl",
 			},
 		},
 		{
 			name:    "with wrong path to message template",
-			errText: "can't read message template: open notfound.tmpl: no such file or directory",
+			errText: "notfound.tmpl: file does not exist",
 			emailParams: EmailParams{
-				VerificationTemplatePath: "testdata/verification.html.tmpl",
-				MsgTemplatePath:          "notfound.tmpl",
+				MsgTemplatePath: "notfound.tmpl",
 			},
 		},
 		{

backend/app/notify/notify.go

@@ -34,8 +34,8 @@ type Destination interface {
 // Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
-	GetUserEmail(siteID string, userID string) (string, error)
-	GetUserTelegram(siteID string, userID string) (string, error)
+	GetUserEmail(siteID, userID string) (string, error)
+	GetUserTelegram(siteID, userID string) (string, error)
 }
 
 // used for email and telegram retrieval from user details
@@ -143,6 +143,12 @@ func (s *Service) SubmitVerification(req VerificationRequest) {
 // Close queue channel and wait for completion
 func (s *Service) Close() {
 	if s.queue != nil {
+		// don't panic in case service is already closed
+		select {
+		case <-s.ctx.Done():
+			return
+		default:
+		}
 		log.Print("[DEBUG] close notifier")
 		close(s.queue)
 		close(s.verificationQueue)

backend/app/notify/notify_test.go

@@ -21,6 +21,8 @@ func TestService_NoDestinations(t *testing.T) {
 	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
+	// second call should not result in panic
+	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {

backend/app/notify/webhook.go

@@ -3,6 +3,7 @@ package notify
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"text/template"
 	"time"
@@ -12,7 +13,7 @@ import (
 )
 
 const (
-	webhookDefaultTemplate = `{"text": "{{.Text}}"}`
+	webhookDefaultTemplate = `{"text": {{.Text | escapeJSONString}}}`
 )
 
 // WebhookParams contain settings for webhook notifications
@@ -49,7 +50,7 @@ func NewWebhook(params WebhookParams) (*Webhook, error) {
 		params.Template = webhookDefaultTemplate
 	}
 
-	payloadTmpl, err := template.New("webhook").Parse(params.Template)
+	payloadTmpl, err := template.New("webhook").Funcs(template.FuncMap{"escapeJSONString": escapeJSONString}).Parse(params.Template)
 	if err != nil {
 		return nil, fmt.Errorf("unable to parse webhook template: %w", err)
 	}
@@ -82,3 +83,12 @@ func (w *Webhook) SendVerification(_ context.Context, _ VerificationRequest) err
 func (w *Webhook) String() string {
 	return fmt.Sprintf("%s to %s", w.Webhook.String(), w.url)
 }
+
+// escapeJSONString escapes string for JSON
+func escapeJSONString(s string) (string, error) {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

backend/app/notify/webhook_test.go

@@ -2,6 +2,9 @@ package notify
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -34,6 +37,32 @@ func TestWebhook_NewWebhook(t *testing.T) {
 	assert.Contains(t, err.Error(), "unable to parse webhook template")
 }
 
+// https://github.com/umputun/remark42/issues/1791
+func TestWebhook_ReceiveValidJSON(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/webhook-notify")
+		assert.Equal(t, "POST", r.Method)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		t.Log("received body", string(body))
+		assert.JSONEq(t, `{"text": "<p>testme</p>\n"}`, string(body))
+	}))
+	defer ts.Close()
+
+	wh, err := NewWebhook(WebhookParams{
+		URL:     ts.URL + "/webhook-notify",
+		Headers: []string{"Content-Type:application/json,text/plain"},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	f := store.NewCommentFormatter()
+	c := store.Comment{Text: f.FormatText("testme", false), ParentID: "1", ID: "999"}
+
+	err = wh.Send(context.Background(), Request{Comment: c})
+	assert.NoError(t, err)
+}
+
 func TestWebhook_Send(t *testing.T) {
 	wh, err := NewWebhook(WebhookParams{
 		URL:     "bad-url",

backend/app/rest/api/admin.go

@@ -9,7 +9,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 
@@ -29,15 +29,15 @@ type admin struct {
 
 type adminStore interface {
 	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
-	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
-	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	DeleteUser(siteID, userID string, mode store.DeleteMode) error
+	DeleteUserDetail(siteID, userID string, detail engine.UserDetail) error
 	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
-	IsBlocked(siteID string, userID string) bool
-	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
+	IsBlocked(siteID, userID string) bool
+	SetBlock(siteID, userID string, status bool, ttl time.Duration) error
 	BlockedUsers(siteID string) ([]store.BlockedUser, error)
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
-	SetVerified(siteID string, userID string, status bool) error
+	SetVerified(siteID, userID string, status bool) error
 	SetReadOnly(locator store.Locator, status bool) error
 	SetPin(locator store.Locator, commentID string, status bool) error
 }

backend/app/rest/api/admin_test.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
 	"github.com/stretchr/testify/assert"
@@ -111,7 +111,7 @@ func TestAdmin_Title(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"127.0.0.1"})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
 			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
@@ -348,7 +348,7 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
 	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
-	srv.pubRest.cache = cache.NewScache(cache.NewNopCache()) // TODO: with lru cache it won't be refreshed and invalidated for long
+	srv.pubRest.cache = cache.NewScache[[]byte](cache.NewNopCache[[]byte]()) // TODO: with lru cache it won't be refreshed and invalidated for long
 	// time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
@@ -513,6 +513,7 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.Error(t, err)
 
+	// test format "tree"
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	comments := commentsWithInfo{}
@@ -521,6 +522,16 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 	assert.True(t, comments.Info.ReadOnly)
 	t.Logf("%+v", comments)
+
+	// test format "plain"
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah")
+	assert.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
 }
 
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
@@ -743,9 +754,9 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "provider1_user1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "provider1_user2"}}
 
 	_, err := srv.DataService.Create(c1)
 	assert.NoError(t, err)
@@ -768,13 +779,13 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
-			Id:        "1234567",
+			Id:        "provider1_1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID: "user1",
+			ID: "provider1_user1",
 			Attributes: map[string]interface{}{
 				"delete_me": true,
 			},

backend/app/rest/api/migrator.go

@@ -5,14 +5,13 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"sync"
 	"time"
 
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 
@@ -137,7 +136,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
 		w.Header().Set("Content-Type", "application/gzip")
 		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
 		gzWriter := gzip.NewWriter(w)
 		defer func() {
 			if e := gzWriter.Close(); e != nil {
@@ -164,7 +162,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
 		return
 	}
-	defer r.Body.Close()
+	defer r.Body.Close() //nolint gosec // we don't care about response body
 
 	// start remap procedure with mapper
 	go func() {
@@ -172,7 +170,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 		defer m.setBusy(siteID, false)
 
 		// do export
-		fh, e := ioutil.TempFile("", "remark42_convert")
+		fh, e := os.CreateTemp("", "remark42_convert")
 		if e != nil {
 			log.Printf("[WARN] failed to make temp file %+v", e)
 			return
@@ -250,7 +248,7 @@ func (m *Migrator) runImport(siteID, provider, tmpfile string) {
 
 // saveTemp reads from reader and saves to temp file
 func (m *Migrator) saveTemp(r io.Reader) (string, error) {
-	tmpfile, err := ioutil.TempFile("", "remark42_import")
+	tmpfile, err := os.CreateTemp("", "remark42_import")
 	if err != nil {
 		return "", fmt.Errorf("can't make temp file: %w", err)
 	}

backend/app/rest/api/migrator_test.go

@@ -9,6 +9,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -49,6 +50,22 @@ func TestMigrator_Import(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
@@ -84,6 +101,22 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
@@ -108,6 +141,22 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 3, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments), "2 comments with 1 reply")
 }
 
 func TestMigrator_ImportFromCommento(t *testing.T) {
@@ -115,7 +164,7 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
-"domain":"example.com","url":"https://example.com/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
+"domain":"example.com","url":"/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z",
 "direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
@@ -137,6 +186,63 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+}
+
+func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	r, err := os.Open("testdata/commento.json")
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
+	assert.NoError(t, err)
+	req.Header.Add("Content-Type", "application/json; charset=utf-8")
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+	assert.NoError(t, resp.Body.Close())
+
+	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 7, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 5, len(comments.Comments), "five comments with two replies")
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
@@ -197,6 +303,20 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
@@ -236,6 +356,14 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_Export(t *testing.T) {
@@ -263,6 +391,16 @@ func TestMigrator_Export(t *testing.T) {
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 
+	// export wrong site, should result in error
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=test", http.NoBody)
+	require.NoError(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+
 	// check file mode
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", http.NoBody)
 	require.NoError(t, err)
@@ -339,6 +477,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
@@ -347,6 +486,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// we want remap urls to another domain - www.remark42.com
@@ -364,6 +504,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
@@ -372,6 +522,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+	require.True(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// should find nothing from previous url
@@ -381,6 +541,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
 	require.Equal(t, http.StatusOK, code)
@@ -388,6 +549,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_RemapReject(t *testing.T) {

backend/app/rest/api/rest.go

@@ -3,25 +3,29 @@ package api
 import (
 	"bytes"
 	"context"
+	"embed"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
+	"net/mail"
+	"os"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/didip/tollbooth/v6"
+	"github.com/didip/tollbooth/v7"
 	"github.com/didip/tollbooth_chi"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/logger"
-	"github.com/rakyll/statik/fs"
 
 	"github.com/umputun/remark42/backend/app/notify"
 	"github.com/umputun/remark42/backend/app/rest"
@@ -29,7 +33,6 @@ import (
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/image"
 	"github.com/umputun/remark42/backend/app/store/service"
-	"github.com/umputun/remark42/backend/app/templates"
 )
 
 // Rest is a rest access server
@@ -48,6 +51,7 @@ type Rest struct {
 
 	AnonVote        bool
 	WebRoot         string
+	WebFS           embed.FS
 	RemarkURL       string
 	ReadOnlyAge     int
 	SharedSecret    string
@@ -55,26 +59,28 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter       float64
-	EmailNotifications  bool
-	TelegramBotUsername string
-	EmojiEnabled        bool
-	SimpleView          bool
-	ProxyCORS           bool
-	SendJWTHeader       bool
-	AllowedAncestors    []string // sets Content-Security-Policy "frame-ancestors ..."
-	SubscribersOnly     bool
-	DisableSignature    bool // prevent signature from being added to headers
+	UpdateLimiter              float64
+	EmailNotifications         bool
+	TelegramNotifications      bool
+	EmojiEnabled               bool
+	SimpleView                 bool
+	ProxyCORS                  bool
+	SendJWTHeader              bool
+	AllowedAncestors           []string // sets Content-Security-Policy "frame-ancestors ..."
+	SubscribersOnly            bool
+	DisableSignature           bool // prevent signature from being added to headers
+	DisableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	pubRest   public
-	privRest  private
-	adminRest admin
-	rssRest   rss
+	pubRest          public
+	privRest         private
+	adminRest        admin
+	rssRest          rss
+	openRouteLimiter float64
 }
 
 // LoadingCache defines interface for caching
@@ -85,7 +91,7 @@ type LoadingCache interface {
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
-
+const openRouteLimiter = 10     // limit for open routes
 const lastCommentsScope = "last"
 
 type commentsWithInfo struct {
@@ -93,6 +99,11 @@ type commentsWithInfo struct {
 	Info     store.PostInfo  `json:"info,omitempty"`
 }
 
+type treeWithInfo struct {
+	*service.Tree
+	Info store.PostInfo `json:"info,omitempty"`
+}
+
 // Run the lister and request's router, activate rest server
 func (s *Rest) Run(address string, port int) {
 	if address == "*" {
@@ -188,6 +199,10 @@ func (s *Rest) makeHTTPServer(address string, port int, router http.Handler) *ht
 }
 
 func (s *Rest) routes() chi.Router {
+	if s.openRouteLimiter == 0 {
+		// set the default open route limiter. Just a safety measure as it should be set by Run method anyway
+		s.openRouteLimiter = openRouteLimiter
+	}
 	router := chi.NewRouter()
 	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
 	if !s.DisableSignature {
@@ -223,7 +238,8 @@ func (s *Rest) routes() chi.Router {
 
 	router.Group(func(r chi.Router) {
 		r.Use(middleware.Timeout(5 * time.Second))
-		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)), middleware.NoCache)
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(2, nil)), middleware.NoCache)
+		r.Use(validEmailAuth()) // reject suspicious email logins
 		r.Mount("/auth", authHandler)
 	})
 
@@ -246,7 +262,7 @@ func (s *Rest) routes() chi.Router {
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(middleware.Timeout(30 * time.Second))
-			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.openRouteLimiter, nil)))
 			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/config", s.configCtrl)
 			ropen.Get("/find", s.pubRest.findCommentsCtrl)
@@ -344,14 +360,13 @@ func (s *Rest) routes() chi.Router {
 	router.Group(func(rroot chi.Router) {
 		rroot.Use(middleware.Timeout(10 * time.Second))
 		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
-		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
 		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
 		rroot.Get("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
 		rroot.Post("/email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
 	})
 
-	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot), s.Version)
+	// file server for static content from s.WebRoot on path /web
+	addFileServer(router, s.WebFS, s.WebRoot, s.Version)
 	return router
 }
 
@@ -362,21 +377,20 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 		imageService:     s.ImageService,
 		commentFormatter: s.CommentFormatter,
 		readOnlyAge:      s.ReadOnlyAge,
-		webRoot:          s.WebRoot,
 	}
 
 	privGrp := private{
-		dataService:      s.DataService,
-		cache:            s.Cache,
-		imageService:     s.ImageService,
-		commentFormatter: s.CommentFormatter,
-		readOnlyAge:      s.ReadOnlyAge,
-		authenticator:    s.Authenticator,
-		notifyService:    s.NotifyService,
-		telegramService:  s.TelegramService,
-		remarkURL:        s.RemarkURL,
-		anonVote:         s.AnonVote,
-		templates:        templates.NewFS(),
+		dataService:                s.DataService,
+		cache:                      s.Cache,
+		imageService:               s.ImageService,
+		commentFormatter:           s.CommentFormatter,
+		readOnlyAge:                s.ReadOnlyAge,
+		authenticator:              s.Authenticator,
+		notifyService:              s.NotifyService,
+		telegramService:            s.TelegramService,
+		remarkURL:                  s.RemarkURL,
+		anonVote:                   s.AnonVote,
+		disableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	admGrp := admin{
@@ -412,44 +426,46 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 	emails, _ := s.DataService.AdminStore.Email(siteID)
 
 	cnf := struct {
-		Version             string   `json:"version"`
-		EditDuration        int      `json:"edit_duration"`
-		AdminEdit           bool     `json:"admin_edit"`
-		MaxCommentSize      int      `json:"max_comment_size"`
-		Admins              []string `json:"admins"`
-		AdminEmail          string   `json:"admin_email"`
-		Auth                []string `json:"auth_providers"`
-		AnonVote            bool     `json:"anon_vote"`
-		LowScore            int      `json:"low_score"`
-		CriticalScore       int      `json:"critical_score"`
-		PositiveScore       bool     `json:"positive_score"`
-		ReadOnlyAge         int      `json:"readonly_age"`
-		MaxImageSize        int      `json:"max_image_size"`
-		EmailNotifications  bool     `json:"email_notifications"`
-		TelegramBotUsername string   `json:"telegram_bot_username"`
-		EmojiEnabled        bool     `json:"emoji_enabled"`
-		SimpleView          bool     `json:"simple_view"`
-		SendJWTHeader       bool     `json:"send_jwt_header"`
-		SubscribersOnly     bool     `json:"subscribers_only"`
+		Version               string   `json:"version"`
+		EditDuration          int      `json:"edit_duration"`
+		AdminEdit             bool     `json:"admin_edit"`
+		MinCommentSize        int      `json:"min_comment_size"`
+		MaxCommentSize        int      `json:"max_comment_size"`
+		Admins                []string `json:"admins"`
+		AdminEmail            string   `json:"admin_email"`
+		Auth                  []string `json:"auth_providers"`
+		AnonVote              bool     `json:"anon_vote"`
+		LowScore              int      `json:"low_score"`
+		CriticalScore         int      `json:"critical_score"`
+		PositiveScore         bool     `json:"positive_score"`
+		ReadOnlyAge           int      `json:"readonly_age"`
+		MaxImageSize          int      `json:"max_image_size"`
+		EmailNotifications    bool     `json:"email_notifications"`
+		TelegramNotifications bool     `json:"telegram_notifications"`
+		EmojiEnabled          bool     `json:"emoji_enabled"`
+		SimpleView            bool     `json:"simple_view"`
+		SendJWTHeader         bool     `json:"send_jwt_header"`
+		SubscribersOnly       bool     `json:"subscribers_only"`
 	}{
-		Version:             s.Version,
-		EditDuration:        int(s.DataService.EditDuration.Seconds()),
-		AdminEdit:           s.DataService.AdminEdits,
-		MaxCommentSize:      s.DataService.MaxCommentSize,
-		Admins:              admins,
-		AdminEmail:          emails,
-		LowScore:            s.ScoreThresholds.Low,
-		CriticalScore:       s.ScoreThresholds.Critical,
-		PositiveScore:       s.DataService.PositiveScore,
-		ReadOnlyAge:         s.ReadOnlyAge,
-		MaxImageSize:        s.ImageService.MaxSize,
-		EmailNotifications:  s.EmailNotifications,
-		TelegramBotUsername: s.TelegramBotUsername,
-		EmojiEnabled:        s.EmojiEnabled,
-		AnonVote:            s.AnonVote,
-		SimpleView:          s.SimpleView,
-		SendJWTHeader:       s.SendJWTHeader,
-		SubscribersOnly:     s.SubscribersOnly,
+		Version:               s.Version,
+		EditDuration:          int(s.DataService.EditDuration.Seconds()),
+		AdminEdit:             s.DataService.AdminEdits,
+		MinCommentSize:        s.DataService.MinCommentSize,
+		MaxCommentSize:        s.DataService.MaxCommentSize,
+		Admins:                admins,
+		AdminEmail:            emails,
+		LowScore:              s.ScoreThresholds.Low,
+		CriticalScore:         s.ScoreThresholds.Critical,
+		PositiveScore:         s.DataService.PositiveScore,
+		ReadOnlyAge:           s.ReadOnlyAge,
+		MaxImageSize:          s.ImageService.MaxSize,
+		EmailNotifications:    s.EmailNotifications,
+		TelegramNotifications: s.TelegramNotifications,
+		EmojiEnabled:          s.EmojiEnabled,
+		AnonVote:              s.AnonVote,
+		SimpleView:            s.SimpleView,
+		SendJWTHeader:         s.SendJWTHeader,
+		SubscribersOnly:       s.SubscribersOnly,
 	}
 
 	cnf.Auth = []string{}
@@ -464,34 +480,28 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, cnf)
 }
 
-// serves static files from /web or embedded by statik
-func addFileServer(r chi.Router, path string, root http.FileSystem, version string) {
+// serves static files from the webRoot directory or files embedded into the compiled binary if that directory is absent
+func addFileServer(r chi.Router, embedFS embed.FS, webRoot, version string) {
 	var webFS http.Handler
 
-	statikFS, err := fs.New()
-	if err != nil {
-		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
-		log.Printf("[INFO] run file server for %s, path %s", root, path)
-		webFS = http.FileServer(root)
+	if _, err := os.Stat(webRoot); err == nil {
+		log.Printf("[INFO] run file server from %s from the disk", webRoot)
+		webFS = http.FileServer(http.Dir(webRoot))
 	} else {
-		log.Printf("[INFO] run file server for %s, embedded", root)
-		webFS = http.FileServer(statikFS)
+		log.Printf("[INFO] run file server, embedded")
+		var contentFS, _ = fs.Sub(embedFS, "web")
+		webFS = http.FileServer(http.FS(contentFS))
 	}
 
-	origPath := path
-	webFS = http.StripPrefix(path, webFS)
-	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", http.StatusMovedPermanently).ServeHTTP)
-		path += "/"
-	}
-	path += "*"
+	webFS = http.StripPrefix("/web", webFS)
+	r.Get("/web", http.RedirectHandler("/web/", http.StatusMovedPermanently).ServeHTTP)
 
 	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)),
 		middleware.Timeout(10*time.Second),
 		cacheControl(time.Hour, version),
-	).Get(path, func(w http.ResponseWriter, r *http.Request) {
+	).Get("/web/*", func(w http.ResponseWriter, r *http.Request) {
 		// don't show dirs, just serve files
-		if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
+		if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != ("/web/") {
 			http.NotFound(w, r)
 			return
 		}
@@ -650,6 +660,52 @@ func subscribersOnly(enable bool) func(http.Handler) http.Handler {
 	}
 }
 
+// validEmailAuth is a middleware for auth endpoints for email method.
+// it rejects login request if user, site or email are suspicious
+func validEmailAuth() func(http.Handler) http.Handler {
+
+	reUser := regexp.MustCompile(`^[\p{L}\d\s_]{4,64}$`) // matches ui side validation, adding min/max limitation
+	reSite := regexp.MustCompile(`^[a-zA-Z\d\s_.-]{1,64}$`)
+
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+
+			if r.URL.Path != "/auth/email/login" {
+				// not email login, skip the check
+				h.ServeHTTP(w, r)
+				return
+			}
+
+			if u := r.URL.Query().Get("user"); u != "" {
+				if !reUser.MatchString(u) {
+					log.Printf("[WARN] suspicious user rejected: %s", u)
+					http.Error(w, "Access denied", http.StatusForbidden)
+					return
+				}
+			}
+
+			if a := r.URL.Query().Get("address"); a != "" {
+				if _, err := mail.ParseAddress(a); err != nil {
+					log.Printf("[WARN] suspicious address rejected: %s", a)
+					http.Error(w, "Access denied", http.StatusForbidden)
+					return
+				}
+			}
+
+			if s := r.URL.Query().Get("site"); s != "" {
+				if !reSite.MatchString(s) {
+					log.Printf("[WARN] suspicious site rejected: %s", s)
+					http.Error(w, "Access denied", http.StatusForbidden)
+					return
+				}
+			}
+
+			h.ServeHTTP(w, r)
+		}
+		return http.HandlerFunc(fn)
+	}
+}
+
 func parseError(err error, defaultCode int) (code int) {
 	code = defaultCode
 

backend/app/rest/api/rest_private.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"crypto/sha1" //nolint:gosec //not used for security
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"io"
@@ -17,7 +18,7 @@ import (
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
@@ -33,17 +34,17 @@ import (
 )
 
 type private struct {
-	dataService      privStore
-	cache            LoadingCache
-	readOnlyAge      int
-	commentFormatter *store.CommentFormatter
-	imageService     *image.Service
-	notifyService    *notify.Service
-	authenticator    *auth.Service
-	telegramService  telegramService
-	remarkURL        string
-	anonVote         bool
-	templates        templates.FileReader
+	dataService                privStore
+	cache                      LoadingCache
+	readOnlyAge                int
+	commentFormatter           *store.CommentFormatter
+	imageService               *image.Service
+	notifyService              *notify.Service
+	authenticator              *auth.Service
+	telegramService            telegramService
+	remarkURL                  string
+	anonVote                   bool
+	disableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 }
 
 // telegramService is a subset of Telegram service used for setting up user telegram notifications
@@ -59,15 +60,15 @@ type privStore interface {
 	Vote(req service.VoteReq) (comment store.Comment, err error)
 	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
 	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
-	GetUserEmail(siteID string, userID string) (string, error)
-	SetUserEmail(siteID string, userID string, value string) (string, error)
-	GetUserTelegram(siteID string, userID string) (string, error)
-	SetUserTelegram(siteID string, userID string, value string) (string, error)
-	DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error
+	GetUserEmail(siteID, userID string) (string, error)
+	SetUserEmail(siteID, userID, value string) (string, error)
+	GetUserTelegram(siteID, userID string) (string, error)
+	SetUserTelegram(siteID, userID, value string) (string, error)
+	DeleteUserDetail(siteID, userID string, detail engine.UserDetail) error
 	ValidateComment(c *store.Comment) error
-	IsVerified(siteID string, userID string) bool
+	IsVerified(siteID, userID string) bool
 	IsReadOnly(locator store.Locator) bool
-	IsBlocked(siteID string, userID string) bool
+	IsBlocked(siteID, userID string) bool
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 }
 
@@ -88,14 +89,14 @@ func (s *private) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 	comment.Sanitize()
 
-	// check if images are valid
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	// check if images are valid, omit proxied images as they are lazy-loaded
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		err := s.imageService.ResetCleanupTimer(id)
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't renew staged picture cleanup timer", rest.ErrImgNotFound)
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
 			return
 		}
 	}
@@ -128,10 +129,10 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 
-	// check if images are valid
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	// check if images are valid, omit proxied images as they are lazy-loaded
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		_, err := s.imageService.Load(id)
 		if err != nil {
 			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
@@ -151,7 +152,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	id, err := s.dataService.Create(comment)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentRestrictWords)
 		return
 	}
@@ -173,7 +174,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		s.notifyService.Submit(notify.Request{Comment: finalComment})
 	}
 
-	log.Printf("[DEBUG] created commend %+v", finalComment)
+	log.Printf("[DEBUG] created comment %+v", finalComment)
 
 	render.Status(r, http.StatusCreated)
 	render.JSON(w, r, &finalComment)
@@ -188,7 +189,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't read comment details from body", rest.ErrDecode)
 		return
 	}
 
@@ -212,7 +213,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.commentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text, s.disableFancyTextFormatting),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
@@ -220,7 +221,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res, err := s.dataService.EditComment(locator, id, editReq)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
@@ -245,7 +246,7 @@ func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
 		}
-		if len(email) > 0 {
+		if email != "" {
 			user.EmailSubscription = true
 		}
 	}
@@ -308,28 +309,59 @@ func (s *private) getEmailCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // sendEmailConfirmationCtrl gets address and siteID from query, makes confirmation token and sends it to user.
-// GET /email/subscribe?site=siteID&address=someone@example.com
+// In case user is logged in with the same email, and auto_confirm is true, confirm it right away.
+// In case of quick confirmation, "updated" is set to true, otherwise - to false.
+// POST /email/subscribe with site and address in json body
+//
 //nolint:dupl // too hard to deduplicate that logic, as then it's tricky to use SendErrorJSON
 func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
-	address := r.URL.Query().Get("address")
-	siteID := r.URL.Query().Get("site")
-	if address == "" {
+
+	subscribe := struct {
+		Site        string
+		Address     string
+		autoConfirm bool
+	}{autoConfirm: true}
+	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &subscribe); err != nil {
+		if err != io.EOF {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse request body", rest.ErrDecode)
+			return
+		}
+		// old behavior fallback, reading from the query params. Auto confirm is false in this case.
+		subscribe.Address = r.URL.Query().Get("address")
+		subscribe.Site = r.URL.Query().Get("site")
+		subscribe.autoConfirm = false
+	}
+
+	if subscribe.Address == "" {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest,
 			fmt.Errorf("missing parameter"), "address parameter is required", rest.ErrInternal)
 		return
 	}
-	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
-	if err != nil {
-		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	existingAddress, getErr := s.dataService.GetUserEmail(subscribe.Site, user.ID)
+	if getErr != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, getErr)
 	}
-	if address == existingAddress {
+	if subscribe.Address == existingAddress {
 		rest.SendErrorJSON(w, r, http.StatusConflict,
 			fmt.Errorf("already verified"), "email address is already verified for this user", rest.ErrInternal)
 		return
 	}
+
+	// in case the user logged in with the same email as they try to subscribe with, confirm it right away
+	// this behavior is different from the previous one and is hidden behind the autoConfirm flag,
+	// which is true for the new API, and false for the old one
+	//
+	// nolint:gosec // this is not used for security purposes
+	if subscribe.autoConfirm &&
+		strings.HasPrefix(user.ID, "email_") &&
+		strings.TrimPrefix(user.ID, "email_") == token.HashID(sha1.New(), subscribe.Address) {
+		s.setEmail(w, r, user.ID, subscribe.Site, subscribe.Address)
+		return
+	}
+
 	claims := token.Claims{
-		Handshake: &token.Handshake{ID: user.ID + "::" + address},
+		Handshake: &token.Handshake{ID: user.ID + "::" + subscribe.Address},
 		StandardClaims: jwt.StandardClaims{
 			Audience:  r.URL.Query().Get("site"),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
@@ -346,14 +378,14 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
 
 	s.notifyService.SubmitVerification(
 		notify.VerificationRequest{
-			SiteID: siteID,
+			SiteID: subscribe.Site,
 			User:   user.Name,
-			Email:  address,
+			Email:  subscribe.Address,
 			Token:  tkn,
 		},
 	)
 
-	render.JSON(w, r, R.JSON{"user": user, "address": address})
+	render.JSON(w, r, R.JSON{"user": user, "address": subscribe.Address, "updated": false})
 }
 
 // telegramSubscribeCtrl generates and verifies telegram notification request
@@ -400,7 +432,7 @@ func (s *private) telegramSubscribeCtrl(w http.ResponseWriter, r *http.Request)
 	var address, siteID string
 	address, siteID, err := s.telegramService.CheckToken(queryToken, user.ID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set telegram for user", rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusNotFound, err, "request is not verified yet", rest.ErrInternal)
 		return
 	}
 
@@ -417,17 +449,29 @@ func (s *private) telegramSubscribeCtrl(w http.ResponseWriter, r *http.Request)
 }
 
 // setConfirmedEmailCtrl uses provided token parameter (generated by sendEmailConfirmationCtrl) to set email and add it to user token
-// PUT /email/confirm?site=siteID&tkn=jwt
+// POST /email/confirm with site and token in json body
 func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) {
-	tkn := r.URL.Query().Get("tkn")
-	if tkn == "" {
+	user := rest.MustGetUserInfo(r)
+
+	confirm := struct {
+		Site  string
+		Token string
+	}{}
+	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &confirm); err != nil {
+		if err != io.EOF {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse request body", rest.ErrDecode)
+			return
+		}
+		// old behavior fallback, reading from the query params
+		confirm.Token = r.URL.Query().Get("tkn")
+		confirm.Site = r.URL.Query().Get("site")
+	}
+
+	if confirm.Token == "" {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal)
 		return
 	}
-	user := rest.MustGetUserInfo(r)
-	siteID := r.URL.Query().Get("site")
-
-	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	confClaims, err := s.authenticator.TokenService().Parse(confirm.Token)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
 		return
@@ -445,17 +489,20 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	address := elems[1]
+	s.setEmail(w, r, user.ID, confirm.Site, address)
+}
 
-	log.Printf("[DEBUG] set email for user %s", user.ID)
+func (s *private) setEmail(w http.ResponseWriter, r *http.Request, userID, siteID, address string) {
+	log.Printf("[DEBUG] set email for user %s", userID)
 
-	val, err := s.dataService.SetUserEmail(siteID, user.ID, address)
+	val, err := s.dataService.SetUserEmail(siteID, userID, address)
 	if err != nil {
 		code := parseError(err, rest.ErrInternal)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set email for user", code)
 		return
 	}
 
-	// update User.Email from the token
+	// update User.Email field
 	claims, _, err := s.authenticator.TokenService().Get(r)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
@@ -473,29 +520,26 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request)
 func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 	tkn := r.URL.Query().Get("tkn")
 	if tkn == "" {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest,
-			fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal)
 		return
 	}
 	siteID := r.URL.Query().Get("site")
 
 	confClaims, err := s.authenticator.TokenService().Parse(tkn)
 	if err != nil {
-		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
 		return
 	}
 
 	if s.authenticator.TokenService().IsExpired(confClaims) {
-		rest.SendErrorHTML(w, r, http.StatusForbidden,
-			fmt.Errorf("expired"), "failed to verify confirmation token", rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusForbidden, fmt.Errorf("expired"), "failed to verify confirmation token", rest.ErrInternal)
 		return
 	}
 
 	// Handshake.ID is user.ID + "::" + address
 	elems := strings.Split(confClaims.Handshake.ID, "::")
 	if len(elems) != 2 {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest,
-			fmt.Errorf("%s", confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("%s", confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
 		return
 	}
 	userID := elems[0]
@@ -508,14 +552,11 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't read email for %s, %v", userID, err)
 	}
 	if existingAddress == "" {
-		rest.SendErrorHTML(w, r, http.StatusConflict,
-			fmt.Errorf("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusConflict, fmt.Errorf("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
 		return
 	}
 	if address != existingAddress {
-		rest.SendErrorHTML(w, r, http.StatusBadRequest,
-			fmt.Errorf("wrong email unsubscription"), "email address in request does not match known for this user",
-			rest.ErrInternal, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
 		return
 	}
 
@@ -523,7 +564,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 
 	if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
 		code := parseError(err, rest.ErrInternal)
-		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code, s.templates)
+		rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
 		return
 	}
 	// clean User.Email from the token, if user has the token
@@ -534,7 +575,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 	if claims.User != nil && claims.User.Email != "" {
 		claims.User.Email = ""
 		if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
-			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal, s.templates)
+			rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
 			return
 		}
 	}
@@ -546,13 +587,13 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	MustRead := func(path string) string {
-		file, err := s.templates.ReadFile(path)
+		file, err := templates.Read(path)
 		if err != nil {
 			panic(err)
 		}
 		return string(file)
 	}
-	tmplstr := MustRead("unsubscribe.html.tmpl")
+	tmplstr := MustRead("email_unsubscribe.html.tmpl")
 	tmpl := template.Must(template.New("unsubscribe").Parse(tmplstr))
 	msg := bytes.Buffer{}
 	MustExecute(tmpl, &msg, nil)

backend/app/rest/api/rest_private_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
@@ -24,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/image"
 )
@@ -54,6 +56,111 @@ func TestRest_Create(t *testing.T) {
 	assert.True(t, len(c["id"].(string)) > 8)
 }
 
+// based on issue https://github.com/umputun/remark42/issues/1292
+func TestRest_CreateFilteredCode(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	resp, err := post(t, ts.URL+"/api/v1/comment",
+		`{"text": "`+"`foo<bar>`"+`", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	assert.NoError(t, err)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+	assert.NoError(t, resp.Body.Close())
+
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	require.NoError(t, err, string(b))
+	loc := c["locator"].(map[string]interface{})
+	assert.Equal(t, "remark42", loc["site"])
+	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
+	assert.Equal(t, "`foo<bar>`", c["orig"])
+	assert.Contains(t, c["text"], "foo")
+	assert.Contains(t, c["text"], "bar")
+	assert.True(t, len(c["id"].(string)) > 8)
+}
+
+// based on issue https://github.com/umputun/remark42/issues/1631
+func TestRest_CreateAndPreviewWithImage(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	ts.Close()
+	defer teardown()
+
+	srv.ImageService.ProxyAPI = srv.RemarkURL + "/api/v1/img"
+	srv.ImageProxy = &proxy.Image{
+		HTTP2HTTPS:    true,
+		CacheExternal: true,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     srv.RemarkURL,
+		ImageService:  srv.ImageService,
+	}
+	srv.CommentFormatter = store.NewCommentFormatter(srv.ImageProxy)
+	// need to recreate the server with new ImageProxy, otherwise old one will be used
+	ts = httptest.NewServer(srv.routes())
+	defer ts.Close()
+
+	var pngRead bool
+	// server with the test PNG image
+	pngServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, e := io.Copy(w, gopherPNG())
+		assert.NoError(t, e)
+		pngRead = true
+	}))
+	defer pngServer.Close()
+
+	t.Run("create", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/comment",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		c := R.JSON{}
+		err = json.Unmarshal(b, &c)
+		require.NoError(t, err, string(b))
+		assert.NotContains(t, c["text"], pngServer.URL)
+		assert.Contains(t, c["text"], srv.RemarkURL)
+		loc := c["locator"].(map[string]interface{})
+		assert.Equal(t, "remark42", loc["site"])
+		assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
+		assert.True(t, len(c["id"].(string)) > 8)
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+	})
+
+	t.Run("preview", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/preview",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		assert.NotContains(t, string(b), pngServer.URL)
+		assert.Contains(t, string(b), srv.RemarkURL)
+
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+		// retrieve the image from the cache
+		imgURL := strings.Split(strings.Split(string(b), "src=\"")[1], "\"")[0]
+		// replace srv.RemarkURL with ts.URL
+		imgURL = strings.ReplaceAll(imgURL, srv.RemarkURL, ts.URL)
+		resp, err = http.Get(imgURL)
+		assert.NoError(t, err)
+		b, err = io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+		// compare image to original gopher png after decoding from base64
+		assert.Equal(t, gopher, base64.StdEncoding.EncodeToString(b))
+
+		assert.Equal(t, true, pngRead, "original image accessed to be shown to user")
+	})
+
+}
+
 func TestRest_CreateOldPost(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -141,6 +248,25 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
 	assert.Equal(t, "invalid comment", c["details"])
 }
 
+func TestRest_CreateRelativeURL(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// check that it's not possible to click insert URL button and not alter the URL in it (which is `url` by default)
+	relativeURLText := `{"text": "here is a link with relative URL: [google.com](url)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
+	resp, err := post(t, ts.URL+"/api/v1/comment", relativeURLText)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	assert.Equal(t, "links should start with mailto:, http:// or https://", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_CreateRejected(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -242,6 +368,56 @@ func TestRest_CreateAndGet(t *testing.T) {
 	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false, IP: ""}, comment.User, "no ip")
 }
 
+func TestRest_CreateWithQuotes(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create comment with quotes with smartypants
+	resp, err := post(t, ts.URL+"/api/v1/comment",
+		`{"text": "smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id := c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment := store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>smartpants «quoted» text</p>\n", comment.Text)
+	assert.Equal(t, "smartpants \"quoted\" text", comment.Orig)
+
+	// create comment with quotes without smartypants
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/comment",
+		`{"text": "no_smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c = R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id = c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment = store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>no_smartpants &#34;quoted&#34; text</p>\n", comment.Text)
+	assert.Equal(t, "no_smartpants \"quoted\" text", comment.Orig)
+}
+
 func TestRest_Update(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -348,6 +524,109 @@ func TestRest_UpdateDelete(t *testing.T) {
 		{URL: "https://radio-t.com/blah2", Count: 0}}, j)
 }
 
+func TestRest_DeleteChildThenParent(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	p := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	idP := addComment(t, p, ts)
+
+	c1 := store.Comment{Text: "test test #1", ParentID: idP,
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	idC1 := addComment(t, c1, ts)
+
+	c2 := store.Comment{Text: "test test #1", ParentID: idP,
+		Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah1"}}
+	idC2 := addComment(t, c2, ts)
+
+	// check multi count equals two
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3}}, j)
+
+	// update a parent comment fails after child is created
+	client := http.Client{}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+idP+"?site=remark42&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"text": "updated text", "summary":"updated by user"}`))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err := client.Do(req)
+	require.NoError(t, err)
+	body, err := io.ReadAll(b.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusBadRequest, b.StatusCode, string(body))
+	assert.NoError(t, b.Body.Close())
+
+	// delete first child comment
+	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+idC1+"?site=remark42&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(b.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, b.StatusCode, string(body))
+	assert.NoError(t, b.Body.Close())
+
+	// delete a parent comment, fails as one comment child still present
+	defer client.CloseIdleConnections()
+	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+idP+"?site=remark42&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(b.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusBadRequest, b.StatusCode, string(body))
+	assert.NoError(t, b.Body.Close())
+
+	// delete second child comment, as an admin to check both deletion methods
+	req, err = http.NewRequest(http.MethodDelete,
+		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, idC2), http.NoBody)
+	require.NoError(t, err)
+	requireAdminOnly(t, req)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// delete a parent comment, shouldn't fail after children are deleted
+	defer client.CloseIdleConnections()
+	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+idP+"?site=remark42&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(b.Body)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, b.StatusCode, string(body))
+	assert.NoError(t, b.Body.Close())
+
+	// check multi count decremented to zero
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah1"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0}}, j)
+}
+
 func TestRest_UpdateNotOwner(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -371,8 +650,6 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	assert.Equal(t, http.StatusForbidden, b.StatusCode, string(body), "update from non-owner")
 	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
-	client = http.Client{}
-	defer client.CloseIdleConnections()
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
 		"?site=remark42&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
 	assert.NoError(t, err)
@@ -448,7 +725,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=remark42&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), http.NoBody)
 		assert.NoError(t, err)
-		req.Header.Add("X-JWT", devToken)
+		req.Header.Add("X-JWT", dev2Token)
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
 		assert.NoError(t, resp.Body.Close())
@@ -457,7 +734,7 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, http.StatusOK, vote(1), "first vote allowed")
 	assert.Equal(t, http.StatusBadRequest, vote(1), "second vote rejected")
-	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
@@ -468,7 +745,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -477,7 +754,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, 0, cr.Vote)
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed one more time")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -486,7 +763,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, -1, cr.Vote)
 
 	assert.Equal(t, http.StatusBadRequest, vote(-1), "dbl vote not allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -575,22 +852,15 @@ func TestRest_AnonVote(t *testing.T) {
 	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs)
 }
 
-type MockFS struct{}
-
-func (fs *MockFS) ReadFile(path string) ([]byte, error) {
-	return []byte(fmt.Sprintf("template %s", path)), nil
-}
-
 func TestRest_EmailAndTelegram(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.privRest.templates = &MockFS{}
 	srv.privRest.telegramService = &mockTelegram{site: "remark42"}
 
 	// issue good token
 	claims := token.Claims{
-		Handshake: &token.Handshake{ID: "dev::good@example.com"},
+		Handshake: &token.Handshake{ID: "provider1_dev::good@example.com"},
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
 			ExpiresAt: time.Now().Add(10 * time.Minute).Unix(),
@@ -609,19 +879,25 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 		responseCode int
 		noAuth       bool
 		cookieEmail  string
+		body         string
 	}{
 		{description: "issue delete request without auth", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusUnauthorized, noAuth: true},
 		{description: "issue delete request without site_id", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusBadRequest},
 		{description: "delete non-existent user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
-		{description: "set user email, token not set", url: "/api/v1/email/confirm?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
-		{description: "send email confirmation without address", url: "/api/v1/email/subscribe?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
-		{description: "send email confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
-		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "set user email, token not set", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusBadRequest, body: `{"site":"remark42"}`},
+		{description: "set user email, token not set, old query param", url: "/api/v1/email/confirm?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send email confirmation without address", url: "/api/v1/email/subscribe", method: http.MethodPost, responseCode: http.StatusBadRequest, body: `{"site":"remark42"}`},
+		{description: "send email confirmation without address, old query param", url: "/api/v1/email/subscribe?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send email confirmation", url: "/api/v1/email/subscribe", method: http.MethodPost, responseCode: http.StatusOK, body: `{"site":"remark42","address":"good@example.com"}`},
+		{description: "send email confirmation, old query param", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
+		{description: "set user email, token is good", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com", body: fmt.Sprintf(`{"site":"remark42","token":%q}`, goodToken)},
+		{description: "set user email, token is good, old query param", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
 		{description: "send confirmation with same address", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusConflict},
 		{description: "get user email", url: "/api/v1/email?site=remark42", method: http.MethodGet, responseCode: http.StatusOK},
 		{description: "delete user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
 		{description: "send another confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
-		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "set user email, token is good", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com", body: fmt.Sprintf(`{"site":"remark42","token":%q}`, goodToken)},
+		{description: "set user email, token is good, old query param", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
 		{description: "unsubscribe user, no token", url: "/email/unsubscribe.html?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
 		{description: "unsubscribe user, wrong token", url: "/email/unsubscribe.html?site=remark42&tkn=jwt", method: http.MethodGet, responseCode: http.StatusForbidden},
 		{description: "unsubscribe user, good token", url: fmt.Sprintf("/email/unsubscribe.html?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK},
@@ -642,7 +918,11 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 	for _, x := range testData {
 		x := x
 		t.Run(x.description, func(t *testing.T) {
-			req, err := http.NewRequest(x.method, ts.URL+x.url, http.NoBody)
+			reqBody := io.NopCloser(strings.NewReader(x.body))
+			if x.body == "" {
+				reqBody = http.NoBody
+			}
+			req, err := http.NewRequest(x.method, ts.URL+x.url, reqBody)
 			require.NoError(t, err)
 			if !x.noAuth {
 				req.Header.Add("X-JWT", devToken)
@@ -679,7 +959,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -718,7 +998,11 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.Empty(t, mockDestination.Get()[1].Emails)
 
 	// send confirmation token for email
-	req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe",
+		io.NopCloser(strings.NewReader(`{"site": "remark42", "address": "good@example.com"}`)),
+	)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -733,8 +1017,31 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.Equal(t, "good@example.com", mockDestination.GetVerify()[0].Email)
 	verificationToken := mockDestination.GetVerify()[0].Token
 
+	// get user information to verify lack of the subscription
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var clearUser store.User
+	err = json.Unmarshal(body, &clearUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: false,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, clearUser)
+
 	// verify email
-	req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/confirm",
+		io.NopCloser(strings.NewReader(fmt.Sprintf(`{"site": "remark42", "token": %q}`, verificationToken))),
+	)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -745,7 +1052,10 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 
 	// get user information to verify the subscription
-	req, err = http.NewRequest(http.MethodGet, ts.URL+"/api/v1/user?site=remark42", http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -754,11 +1064,11 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
-	var user store.User
-	err = json.Unmarshal(body, &user)
+	var subscribedUser store.User
+	err = json.Unmarshal(body, &subscribedUser)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", EmailSubscription: true,
-		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: true,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, subscribedUser)
 
 	// create child comment from another user, email notification expected
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
@@ -809,6 +1119,80 @@ func TestRest_EmailNotification(t *testing.T) {
 	time.Sleep(time.Millisecond * 30)
 	require.Equal(t, 4, len(mockDestination.Get()))
 	assert.Empty(t, mockDestination.Get()[3].Emails)
+
+	// confirm email via subscribe call with query params, old behavior, email notification is expected
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com",
+		http.NoBody,
+	)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 2, len(mockDestination.GetVerify()), "verification email was sent")
+
+	// get email user information to verify there is no subscription yet
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var unsubscribedEmailUser store.User
+	err = json.Unmarshal(body, &unsubscribedEmailUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "good@example.com test user", ID: "email_f5dfe9d2e6bd75fc74ea5fabf273b45b5baeb195", EmailSubscription: false,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, unsubscribedEmailUser)
+
+	// confirm email via subscribe call, no email notification is expected
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe",
+		io.NopCloser(strings.NewReader(`{"site": "remark42", "address": "good@example.com"}`)),
+	)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 2, len(mockDestination.GetVerify()), "no new verification email was sent")
+
+	// get email user information to verify the subscription happened without the confirmation call
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var subscribedEmailUser store.User
+	err = json.Unmarshal(body, &subscribedEmailUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "good@example.com test user", ID: "email_f5dfe9d2e6bd75fc74ea5fabf273b45b5baeb195", EmailSubscription: true,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, subscribedEmailUser)
 }
 
 func TestRest_TelegramNotification(t *testing.T) {
@@ -825,7 +1209,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -904,8 +1288,8 @@ func TestRest_TelegramNotification(t *testing.T) {
 	body, err = io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
-	require.Equal(t, http.StatusInternalServerError, resp.StatusCode, string(body))
-	require.Equal(t, `{"code":0,"details":"can't set telegram for user","error":"not verified"}`+"\n", string(body))
+	require.Equal(t, http.StatusNotFound, resp.StatusCode, string(body))
+	require.Equal(t, `{"code":0,"details":"request is not verified yet","error":"not verified"}`+"\n", string(body))
 
 	mockTlgrm.notVerified = false
 
@@ -953,7 +1337,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	var user store.User
 	err = json.Unmarshal(body, &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
 
 	// create child comment from another user, telegram notification expected
@@ -1012,7 +1396,7 @@ func TestRest_UserAllData(t *testing.T) {
 	defer teardown()
 
 	// write 3 comments
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
@@ -1037,13 +1421,13 @@ func TestRest_UserAllData(t *testing.T) {
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
 	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 	ungzBody, err := io.ReadAll(ungzReader)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	strUungzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUungzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 3, strings.Count(strUungzBody, `"text":`), "3 comments inside")
 
 	parsed := struct {
@@ -1053,7 +1437,7 @@ func TestRest_UserAllData(t *testing.T) {
 
 	err = json.Unmarshal(ungzBody, &parsed)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, parsed.Info)
 	assert.Equal(t, 3, len(parsed.Comments))
 
@@ -1069,7 +1453,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 
@@ -1096,7 +1480,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	strUngzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUngzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 51, strings.Count(strUngzBody, `"text":`), "51 comments inside")
 }
 
@@ -1120,12 +1504,12 @@ func TestRest_DeleteMe(t *testing.T) {
 	err = json.Unmarshal(body, &m)
 	assert.NoError(t, err)
 	assert.Equal(t, "remark42", m["site"])
-	assert.Equal(t, "dev", m["user_id"])
+	assert.Equal(t, "provider1_dev", m["user_id"])
 
 	tkn := m["token"]
 	claims, err := srv.Authenticator.TokenService().Parse(tkn)
 	assert.NoError(t, err)
-	assert.Equal(t, "dev", claims.User.ID)
+	assert.Equal(t, "provider1_dev", claims.User.ID)
 	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+tkn, m["link"])
 
 	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), http.NoBody)

backend/app/rest/api/rest_public.go

@@ -7,15 +7,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
-	"path"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/skip2/go-qrcode"
@@ -32,7 +30,6 @@ type public struct {
 	readOnlyAge      int
 	commentFormatter *store.CommentFormatter
 	imageService     *image.Service
-	webRoot          string
 }
 
 type pubStore interface {
@@ -43,7 +40,7 @@ type pubStore interface {
 	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
 	UserCount(siteID, userID string) (int, error)
 	Count(locator store.Locator) (int, error)
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	List(siteID string, limit, skip int) ([]store.PostInfo, error)
 	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
 
 	ValidateComment(c *store.Comment) error
@@ -53,6 +50,8 @@ type pubStore interface {
 
 // GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
+//
+// When `url` parameter is not set (e.g. request is for site-wide comments), does not return deleted comments.
 func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
@@ -80,22 +79,36 @@ func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
 		comments = s.applyView(comments, view)
+
+		var commentsInfo store.PostInfo
+		if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
+			commentsInfo = info
+		}
+
+		if !since.IsZero() { // if since is set, number of comments can be different from total in the DB
+			commentsInfo.Count = 0
+			for _, c := range comments {
+				if !c.Deleted {
+					commentsInfo.Count++
+				}
+			}
+		}
+
+		// post might be readonly without any comments, Info call will fail then and ReadOnly flag should be checked separately
+		if !commentsInfo.ReadOnly && locator.URL != "" && s.dataService.IsReadOnly(locator) {
+			commentsInfo.ReadOnly = true
+		}
+
 		var b []byte
 		switch format {
 		case "tree":
-			tree := service.MakeTree(comments, sort, s.readOnlyAge)
-			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*service.Node{}
+			withInfo := treeWithInfo{Tree: service.MakeTree(comments, sort), Info: commentsInfo}
+			if withInfo.Nodes == nil { // eliminate json nil serialization
+				withInfo.Nodes = []*service.Node{}
 			}
-			if s.dataService.IsReadOnly(locator) {
-				tree.Info.ReadOnly = true
-			}
-			b, e = encodeJSONWithHTML(tree)
+			b, e = encodeJSONWithHTML(withInfo)
 		default:
-			withInfo := commentsWithInfo{Comments: comments}
-			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
-				withInfo.Info = info
-			}
+			withInfo := commentsWithInfo{Comments: comments, Info: commentsInfo}
 			b, e = encodeJSONWithHTML(withInfo)
 		}
 		return b, e
@@ -345,16 +358,6 @@ func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
-func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
-	data, err := os.ReadFile(path.Join(s.webRoot, "getstarted.html"))
-	if err != nil {
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	render.HTML(w, r, string(data))
-}
-
 // GET /robots.txt
 func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
 	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/user",

backend/app/rest/api/rest_public_test.go

@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -76,13 +78,31 @@ func TestRest_Preview(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat",
+		`{"code":20,"details":"can't load picture from the comment",`+
+			`"error":"can't get image stats for dev_user/bad_picture: stat`,
 	)
 	assert.Contains(t,
 		string(b),
 		"/pics-remark42/staging/dev_user/62/bad_picture: no such file or directory\"}\n",
 	)
+
+	// test quotes with and without smartypants
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>«quoted» text</p>\n", string(b))
+
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>&#34;quoted&#34; text</p>\n", string(b))
 }
 
 func TestRest_PreviewWithWrongImage(t *testing.T) {
@@ -97,8 +117,8 @@ func TestRest_PreviewWithWrongImage(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat ",
+		`{"code":20,"details":"can't load picture from the comment",`+
+			`"error":"can't get image stats for dev_user/bad_picture: stat `,
 	)
 	assert.Contains(t,
 		string(b),
@@ -131,10 +151,10 @@ BKT
 	assert.NoError(t, err)
 	assert.Equal(t,
 		`<h1>h1</h1>
-<pre class="chroma"><code><span><span>func TestRest_Preview(t *testing.T) {
-</span></span><span><span>srv, ts := prep(t)
-</span></span><span><span>  require.NotNil(t, srv)
-</span></span><span><span>}
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="k">func</span> <span class="n">TestRest_Preview</span><span class="p">(</span><span class="n">t</span> <span class="o">*</span><span class="n">testing</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="p">{</span>
+</span></span><span class="line"><span class="cl"><span class="n">srv</span><span class="p">,</span> <span class="n">ts</span> <span class="p">:</span><span class="o">=</span> <span class="n">prep</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+</span></span><span class="line"><span class="cl">  <span class="n">require</span><span class="o">.</span><span class="n">NotNil</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">srv</span><span class="p">)</span>
+</span></span><span class="line"><span class="cl"><span class="p">}</span>
 </span></span></code></pre>`,
 		string(b))
 	assert.NoError(t, resp.Body.Close())
@@ -157,7 +177,7 @@ BKT
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	b, err := io.ReadAll(resp.Body)
 	assert.NoError(t, err)
-	assert.Equal(t, `<pre class="chroma"><code><span><span><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+	assert.Equal(t, `<pre class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
 </span></span></code></pre>`, string(b))
 	assert.NoError(t, resp.Body.Close())
 }
@@ -209,7 +229,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -235,7 +255,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.NoError(t, err)
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
@@ -278,7 +298,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -286,7 +306,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = service.Tree{}
+	tree = treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -325,8 +345,8 @@ func TestRest_FindUserView(t *testing.T) {
 	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
-	assert.Equal(t, "dev", comments.Comments[0].User.ID)
-	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[1].User.ID)
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.Equal(t, "", comments.Comments[1].Text)
 
@@ -440,7 +460,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code, "noting for user blah")
 	assert.Equal(t, `{"comments":[],"count":0}`+"\n", comments)
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -459,7 +479,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev&skip=1&limit=2")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev&skip=1&limit=2")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -477,6 +497,203 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 }
 
+func TestRest_FindUserComments_CWE_918(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"radio-t.com"}) // required for extracting the title, bad URL test
+	defer srv.DataService.TitleExtractor.Close()
+	defer teardown()
+
+	backendRequestedArbitraryServer := false
+	arbitraryServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		t.Logf("request received: %+v", r)
+		backendRequestedArbitraryServer = true
+	}))
+	defer arbitraryServer.Close()
+
+	arbitraryURLComment := store.Comment{Text: "arbitrary URL request test",
+		Locator: store.Locator{SiteID: "remark42", URL: arbitraryServer.URL}}
+
+	assert.False(t, backendRequestedArbitraryServer)
+	addComment(t, arbitraryURLComment, ts)
+	assert.False(t, backendRequestedArbitraryServer,
+		"no request is expected to the test server as it's not in the list of the allowed domains for the title extractor")
+
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
+	assert.Equal(t, http.StatusOK, code)
+
+	resp := struct {
+		Comments []store.Comment
+		Count    int
+	}{}
+
+	err := json.Unmarshal([]byte(res), &resp)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(resp.Comments), "should have 2 comments")
+
+	assert.Equal(t, "", resp.Comments[0].PostTitle, "empty from the first post")
+	assert.Equal(t, arbitraryServer.URL, resp.Comments[0].Locator.URL, "arbitrary URL provided by the request")
+}
+
+func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
+	// test that comment counting is consistent between tree and plain formats
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	commentLocator := store.Locator{URL: "test-url", SiteID: "remark42"}
+
+	// vote for comment multiple times
+	setScore := func(locator store.Locator, id string, val int) {
+		abs := func(x int) int {
+			if x < 0 {
+				return -x
+			}
+			return x
+		}
+		for i := 0; i < abs(val); i++ {
+			_, err := srv.DataService.Vote(service.VoteReq{
+				Locator:   locator,
+				CommentID: id,
+				// unique user ID is needed for correct counting of controversial votes
+				UserID: "user" + strconv.Itoa(val) + strconv.Itoa(i),
+				Val:    val > 0,
+			})
+			require.NoError(t, err)
+		}
+	}
+
+	// Adding initial comments (8 to test-url and 1 to another-url) and voting, and delete two of comments to the first post.
+	// With sleep so that at least few millisecond pass between each comment
+	// and later we would be able to use that in "since" filter with millisecond precision
+	ids := make([]string, 9)
+	timestamps := make([]time.Time, 9)
+	c1 := store.Comment{Text: "top-level comment 1", Locator: commentLocator}
+	ids[0], timestamps[0] = addCommentGetCreatedTime(t, c1, ts)
+	// #3 by score
+	setScore(commentLocator, ids[0], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c2 := store.Comment{Text: "top-level comment 2", Locator: commentLocator}
+	ids[1], timestamps[1] = addCommentGetCreatedTime(t, c2, ts)
+	// #2 by score
+	setScore(commentLocator, ids[1], 2)
+	time.Sleep(time.Millisecond * 5)
+
+	c3 := store.Comment{Text: "second-level comment 1", ParentID: ids[0], Locator: commentLocator}
+	ids[2], timestamps[2] = addCommentGetCreatedTime(t, c3, ts)
+	// #1 by score
+	setScore(commentLocator, ids[2], 10)
+	time.Sleep(time.Millisecond * 5)
+
+	c4 := store.Comment{Text: "third-level comment 1", ParentID: ids[2], Locator: commentLocator}
+	ids[3], timestamps[3] = addCommentGetCreatedTime(t, c4, ts)
+	// #5 by score, #1 by controversy
+	setScore(commentLocator, ids[3], 4)
+	setScore(commentLocator, ids[3], -4)
+	time.Sleep(time.Millisecond * 5)
+
+	c5 := store.Comment{Text: "second-level comment 2", ParentID: ids[1], Locator: commentLocator}
+	ids[4], timestamps[4] = addCommentGetCreatedTime(t, c5, ts)
+	// #5 by score, #2 by controversy
+	setScore(commentLocator, ids[4], 2)
+	setScore(commentLocator, ids[4], -3)
+	time.Sleep(time.Millisecond * 5)
+
+	c6 := store.Comment{Text: "third-level comment 2", ParentID: ids[4], Locator: commentLocator}
+	ids[5], timestamps[5] = addCommentGetCreatedTime(t, c6, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[5], 10)
+	setScore(commentLocator, ids[5], -10)
+	time.Sleep(time.Millisecond * 5)
+
+	c7 := store.Comment{Text: "top-level comment 3", Locator: commentLocator}
+	ids[6], timestamps[6] = addCommentGetCreatedTime(t, c7, ts)
+	// #6 by score, #4 by controversy
+	setScore(commentLocator, ids[6], -3)
+	setScore(commentLocator, ids[6], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c8 := store.Comment{Text: "second-level comment 3", ParentID: ids[6], Locator: commentLocator}
+	ids[7], timestamps[7] = addCommentGetCreatedTime(t, c8, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[7], -20)
+
+	c9 := store.Comment{Text: "comment to post 2", Locator: store.Locator{URL: "another-url", SiteID: "remark42"}}
+	ids[8], timestamps[8] = addCommentGetCreatedTime(t, c9, ts)
+	// #7 by score
+	setScore(store.Locator{URL: "another-url", SiteID: "remark42"}, ids[8], -25)
+
+	// delete two comments bringing the total from 9 to 6
+	err := srv.DataService.Delete(commentLocator, ids[7], store.SoftDelete)
+	assert.NoError(t, err)
+	err = srv.DataService.Delete(commentLocator, ids[5], store.HardDelete)
+	assert.NoError(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
+
+	commentLocator.URL = "readonly-test"
+	// set post without comments to read-only
+	assert.NoError(t, srv.DataService.SetReadOnly(commentLocator, true))
+
+	sinceTenSecondsAgo := strconv.FormatInt(time.Now().Add(-time.Second*10).UnixNano()/1000000, 10)
+	sinceTS := make([]string, 9)
+	formattedTS := make([]string, 9)
+	for i, created := range timestamps {
+		sinceTS[i] = strconv.FormatInt(created.UnixNano()/1000000, 10)
+		formattedTS[i] = created.Format(time.RFC3339Nano)
+	}
+	t.Logf("last timestamp: %v", timestamps[7])
+
+	testCases := []struct {
+		params       string
+		expectedBody string
+	}{
+		{"", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=plain", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"format=plain&url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[0], fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[0], fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[1], fmt.Sprintf(`"info":{"count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[1], fmt.Sprintf(`"info":{"url":"test-url","count":5,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[4], fmt.Sprintf(`"info":{"count":3,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[4], fmt.Sprintf(`"info":{"url":"test-url","count":2,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=tree", `"info":{"count":7`},
+		{"format=tree&url=test-url", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=+time", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=+time", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=-score", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=-score", `"info":{"url":"test-url","count":6`},
+		{"sort=+time", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-time", fmt.Sprintf(`"score":1,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[0])},
+		{"sort=+score", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[2])},
+		{"sort=+score&url=test-url", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[2])},
+		{"sort=-score", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-score&url=test-url", fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[6])},
+		{"sort=-time&since=" + sinceTS[4], fmt.Sprintf(`"score":-1,"vote":0,"controversy":2.924017738212866,"time":%q}],"info":{"count":3`, formattedTS[4])},
+		{"sort=-score&since=" + sinceTS[3], fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":4`, formattedTS[8])},
+		{"sort=-score&url=test-url&since=" + sinceTS[3], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":3`, formattedTS[6])},
+		{"sort=+controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":1`, formattedTS[6])},
+		// three comments of which last one deleted and doesn't have controversy so returned last
+		{"sort=-controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":0,"vote":0,"time":%q,"delete":true}],"info":{"url":"test-url","count":1`, formattedTS[7])},
+		// test readonly status for the post without comments
+		{"url=readonly-test", `"info":{"count":0,"read_only":true`},
+		{"format=tree&url=readonly-test", `"info":{"count":0,"read_only":true`},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.params, func(t *testing.T) {
+			url := fmt.Sprintf(ts.URL+"/api/v1/find?site=remark42&%s", tc.params)
+			body, code := get(t, url)
+			assert.Equal(t, http.StatusOK, code)
+			assert.Contains(t, body, tc.expectedBody)
+			t.Log(body)
+			// prevent hit limiter from engaging
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -486,7 +703,7 @@ func TestRest_UserInfo(t *testing.T) {
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", Picture: "http://example.com/pic.png",
 		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 

backend/app/rest/api/rest_test.go

@@ -18,8 +18,9 @@ import (
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -37,11 +38,18 @@ import (
 	"github.com/umputun/remark42/backend/app/store/service"
 )
 
-var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
+// To generate a token, enter one of the tokens here into https://jwt.io, change the secret to one you're using in your test
+// ("secret" in case of startupT), and alter the fields you want to be changed.
+
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.dirTS_ahSF6375sdO2iodm2K2UmRTzQNQMFiHuTQCVs`
+
+var dev2Token = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYyIiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.qsR_PupfjIq7uw0eAuyGV8nsUoMx9v541c9olnRInRQ`
 
 var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
 
-var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
+var emailUserToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6Imdvb2RAZXhhbXBsZS5jb20gdGVzdCB1c2VyIiwiaWQiOiJlbWFpbF9mNWRmZTlkMmU2YmQ3NWZjNzRlYTVmYWJmMjczYjQ1YjViYWViMTk1IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Imdvb2RAZXhhbXBsZS5jb20ifX0.vH2HN1JpuXL8okTJq1A-zGHQ-l2ILcwxvDDEmu2zwks`
+
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJwcm92aWRlcjFfZGV2IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.X-lvnHvBz6VfEbVV4f-bjcZuLY5pYtvEansk_TQMrX8`
 
 var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
 
@@ -60,23 +68,6 @@ func TestRest_FileServer(t *testing.T) {
 	_ = os.Remove(testHTMLFile)
 }
 
-func TestRest_GetStarted(t *testing.T) {
-	ts, _, teardown := startupT(t)
-	defer teardown()
-
-	getStartedHTML := os.TempDir() + "/getstarted.html"
-	err := os.WriteFile(getStartedHTML, []byte("some html blah"), 0o700)
-	assert.NoError(t, err)
-
-	body, code := get(t, ts.URL+"/index.html")
-	assert.Equal(t, http.StatusOK, code)
-	assert.Equal(t, "some html blah", body)
-
-	_ = os.Remove(getStartedHTML)
-	_, code = get(t, ts.URL+"/index.html")
-	assert.Equal(t, http.StatusNotFound, code)
-}
-
 func TestRest_Shutdown(t *testing.T) {
 	srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
 	done := make(chan bool)
@@ -137,7 +128,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -187,7 +178,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}
@@ -203,7 +194,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 }
 
 func TestRest_rejectAnonUser(t *testing.T) {
-	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		fmt.Fprintln(w, "Hello")
 	}))))
 	defer ts.Close()
@@ -313,10 +304,10 @@ func TestRest_cacheControl(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", tt.url, nil)
+			req := httptest.NewRequest("GET", tt.url, http.NoBody)
 			w := httptest.NewRecorder()
 
-			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -341,10 +332,10 @@ func TestRest_frameAncestors(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req := httptest.NewRequest("GET", "http://example.com", http.NoBody)
 			w := httptest.NewRecorder()
 
-			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -375,12 +366,44 @@ func TestRest_subscribersOnly(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req := httptest.NewRequest("GET", "http://example.com", http.NoBody)
 			if tt.setUser {
 				req = token.SetUserInfo(req, tt.user)
 			}
 			w := httptest.NewRecorder()
-			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+			h.ServeHTTP(w, req)
+			resp := w.Result()
+			assert.Equal(t, tt.status, resp.StatusCode)
+			assert.NoError(t, resp.Body.Close())
+		})
+	}
+}
+
+func Test_validEmailAuth(t *testing.T) {
+	tbl := []struct {
+		req    string
+		status int
+	}{
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=someone", http.StatusOK},
+		{"/auth/email/login?site=site-with-dash_and_underscore-and.dot&address=umputun%example.com&user=someone", http.StatusOK},
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=someone+blah", http.StatusOK},
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=Евгений+Умпутун", http.StatusOK},
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=12", http.StatusForbidden},
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=..blah+blah", http.StatusForbidden},
+		{"/auth/email/login?site=remark42&address=umputun%example.com&user=someonelooong+loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong", http.StatusForbidden},
+		{"/auth/twitter/login?site=remark42&address=umputun%example.com&user=..blah+blah", http.StatusOK},
+		{"/auth/email/login?site=remark42&address=umputun%example.com", http.StatusOK},
+		{"/auth/email/login?site=remark42&address=umputun+example.com&user=someone", http.StatusForbidden},
+		{"/auth/email/login?site=bad!site&address=umputun%example.com&user=someone", http.StatusForbidden},
+		{"/auth/email/login?site=loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongsite&address=umputun%example.com&user=someone", http.StatusForbidden},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := httptest.NewRequest("GET", "http://example.com"+tt.req, http.NoBody)
+			w := httptest.NewRecorder()
+			h := validEmailAuth()(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, tt.status, resp.StatusCode)
@@ -415,7 +438,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "remark42"})
 	require.NoError(t, err)
 
-	memCache := cache.NewScache(cache.NewNopCache())
+	memCache := cache.NewScache[[]byte](cache.NewNopCache[[]byte]())
 
 	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
@@ -435,7 +458,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
+			SecretReader: token.SecretFunc(func(string) (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
@@ -464,16 +487,26 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 			Cache:             memCache,
 			KeyStore:          astore,
 		},
-		NotifyService: notify.NopService,
-		EmojiEnabled:  true,
+		NotifyService:    notify.NopService,
+		EmojiEnabled:     true,
+		openRouteLimiter: 100,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
+	// add some providers. Needed because we don't allow users with unlisted providers to authenticate
+	providers := []string{"provider1", "anonymous", "github", "email"}
+	for _, p := range providers {
+		srv.Authenticator.AddDirectProvider(p, provider.CredCheckerFunc(func(_, _ string) (ok bool, err error) {
+			return true, nil
+		}))
+	}
+
 	for _, h := range srvHook {
 		h(srv)
 	}
 
-	ts = httptest.NewServer(srv.routes())
+	routes := srv.routes()
+	ts = httptest.NewServer(routes)
 
 	teardown = func() {
 		ts.Close()
@@ -532,6 +565,20 @@ func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	return string(b), r.StatusCode
 }
 
+func getWithDev2Auth(t *testing.T, url string) (body string, code int) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("GET", url, http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", dev2Token)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	b, err := io.ReadAll(r.Body)
+	assert.NoError(t, err)
+	require.NoError(t, r.Body.Close())
+	return string(b), r.StatusCode
+}
+
 func getWithAdminAuth(t *testing.T, url string) (response string, statusCode int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	defer client.CloseIdleConnections()
@@ -554,7 +601,7 @@ func post(t *testing.T, url, body string) (*http.Response, error) {
 	return client.Do(req)
 }
 
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+func addCommentGetCreatedTime(t *testing.T, c store.Comment, ts *httptest.Server) (id string, created time.Time) {
 	b, err := json.Marshal(c)
 	require.NoError(t, err, "can't marshal comment %+v", c)
 
@@ -574,7 +621,14 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	err = json.Unmarshal(b, &crResp)
 	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
+	created, err = time.Parse(time.RFC3339, crResp["time"].(string))
+	require.NoError(t, err)
+	return crResp["id"].(string), created
+}
+
+func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+	id, _ := addCommentGetCreatedTime(t, c, ts)
+	return id
 }
 
 func requireAdminOnly(t *testing.T, req *http.Request) {
@@ -613,5 +667,9 @@ func waitForHTTPSServerStart(port int) {
 }
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(
+		m,
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/rest/api/rss.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"github.com/gorilla/feeds"
 
@@ -56,7 +56,6 @@ func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-
 	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}

backend/app/rest/api/ssl.go

@@ -111,15 +111,11 @@ func (s *Rest) getRemarkHost() string {
 
 func (s *Rest) makeTLSConfig() *tls.Config {
 	return &tls.Config{
-		PreferServerCipherSuites: true,
 		CipherSuites: []uint16{
 			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
 			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-			// tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
-			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-			// tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
 		},
 		MinVersion: tls.VersionTLS12,
 		CurvePreferences: []tls.CurveID{

backend/app/rest/api/ssl_test.go

@@ -21,7 +21,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -56,7 +56,7 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}

backend/app/rest/api/testdata/commento.json

@@ -0,0 +1,116 @@
+{
+  "version": 1,
+  "comments": [
+    {
+      "commentHex": "e7a2ef4b4aa1414a7ee65a989889aaecd9d5e7e3bca598ea7a967b4dbcaa8e11",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2022-10-25T07:25:46.807555Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "a29e741145daceb4ca5b3e5e279e05b56f73c04703d93b944718ef757e15317f",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-07-26T12:24:55.058552Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-31T11:03:25.403282Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "6d3bb64ff73b5f9d6a959212ffde472a51abf8bdefaa5ed843659796bceef9de",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:23:47.112062Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-23T12:33:03.370182Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "d0ad6f11cf0c5f8e17457a378a6bb789f412c6b7ef7ada4ae06ec8451f7a18aa",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-18T01:18:38.193625Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "098960fd01c1fc7c0d3ea428f52fab97ea5c18aa52f3565bba679224daddc687",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:24:04.639965Z",
+      "direction": 0,
+      "deleted": false
+    }
+  ],
+  "commenters": [
+    {
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "email": "undefined",
+      "name": "blank",
+      "link": "undefined",
+      "photo": "undefined",
+      "provider": "anon",
+      "joinDate": "2022-06-09T15:54:29.865919Z",
+      "isModerator": false,
+      "deleted": false
+    }
+  ]
+}

backend/app/rest/httperrors.go

@@ -48,9 +48,9 @@ type errTmplData struct {
 	Details string
 }
 
-// SendErrorHTML makes html body with provided template and responds with provided http status code,
+// SendErrorHTML makes html body from error_response.html.tmpl template and responds with provided http status code,
 // error code is not included in render as it is intended for UI developers and not for the users
-func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int, t templates.FileReader) {
+func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
 	// MustExecute behaves like template.Execute, but panics if an error occurs.
 	MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
 		if err = tmpl.Execute(wr, data); err != nil {
@@ -58,7 +58,7 @@ func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, e
 		}
 	}
 	MustRead := func(path string) string {
-		file, e := t.ReadFile(path)
+		file, e := templates.Read(path)
 		if e != nil {
 			panic(e)
 		}

backend/app/rest/httperrors_test.go

@@ -36,18 +36,11 @@ func TestSendErrorJSON(t *testing.T) {
 	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
-type MockFS struct{}
-
-func (fs *MockFS) ReadFile(path string) ([]byte, error) {
-	return []byte(fmt.Sprintf("{{.Error}}{{.Details}} %s", path)), nil
-}
-
 func TestSendErrorHTML(t *testing.T) {
-	fs := &MockFS{}
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorHTML(w, r, 500, fmt.Errorf("error 500"), "error details 123456", 987, fs)
+			SendErrorHTML(w, r, 500, fmt.Errorf("error 500"), "error details 123456", 987)
 			return
 		}
 		w.WriteHeader(404)

backend/app/rest/proxy/image.go

@@ -57,7 +57,7 @@ func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]stri
 		return nil, fmt.Errorf("can't create document: %w", err)
 	}
 	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		if im, ok := s.Attr("src"); ok {
 			if imgSrcPred(im) {
 				result = append(result, im)
@@ -98,6 +98,10 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 	if img == nil {
 		img, err = p.downloadImage(context.Background(), imgURL)
 		if err != nil {
+			if strings.Contains(err.Error(), "invalid content type") {
+				rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid content type", rest.ErrImgNotFound)
+				return
+			}
 			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
 			return
 		}
@@ -159,12 +163,17 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error)
 	if err != nil {
 		return nil, fmt.Errorf("can't download image %s: %w", imgURL, err)
 	}
-	defer resp.Body.Close()
+	defer resp.Body.Close() //nolint gosec // we don't care about response body
 
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
 	}
 
+	contentType := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(contentType, "image/") {
+		return nil, fmt.Errorf("invalid content type %s", contentType)
+	}
+
 	imgData, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read image body")

backend/app/rest/proxy/image_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark42/backend/app/store/image"
@@ -95,7 +94,8 @@ func TestImage_Replace(t *testing.T) {
 }
 
 func TestImage_Routes(t *testing.T) {
-	imageStore := image.MockStore{}
+	// no image supposed to be cached
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",
@@ -108,33 +108,45 @@ func TestImage_Routes(t *testing.T) {
 	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
-	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+	t.Run("valid image", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+		assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	})
 
-	// no image supposed to be cached
-	imageStore.On("Load", mock.Anything).Times(2).Return(nil, nil)
+	t.Run("no image", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+	})
 
-	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	t.Run("bad encoding", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 2, len(imageStore.LoadCalls()))
+	})
 
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	t.Run("non-image reference", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte("https://google.com"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 3, len(imageStore.LoadCalls()))
+	})
 }
 
 func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
-	imageStore := image.MockStore{}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		RemarkURL:    "https://demo.remark42.com",
 		RoutePath:    "/api/v1/proxy",
@@ -148,8 +160,6 @@ func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
-	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
-
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
 	assert.NoError(t, resp.Body.Close())
@@ -157,11 +167,18 @@ func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
 	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
 	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
-	imageStore.AssertCalled(t, "Load", mock.Anything)
+	assert.Equal(t, 1, len(imageStore.LoadCalls()))
 }
 
 func TestImage_RoutesCachingImage(t *testing.T) {
-	imageStore := image.MockStore{}
+	imageStore := image.StoreMock{
+		LoadFunc: func(string) ([]byte, error) {
+			return nil, nil
+		},
+		SaveFunc: func(string, []byte) error {
+			return nil
+		},
+	}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
@@ -177,9 +194,6 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 	imgURL := httpSrv.URL + "/image/img1.png"
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(imgURL))
 
-	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
-	imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
-
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
 	assert.NoError(t, resp.Body.Close())
@@ -187,12 +201,18 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
 	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
 
-	imageStore.AssertCalled(t, "Load", mock.Anything)
-	imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
+	assert.Equal(t, 1, len(imageStore.LoadCalls()))
+	assert.Equal(t, 1, len(imageStore.SaveCalls()))
+	assert.Equal(t, "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), imageStore.SaveCalls()[0].ID)
+	assert.Equal(t, gopherPNGBytes(), imageStore.SaveCalls()[0].Img)
 }
 
 func TestImage_RoutesUsingCachedImage(t *testing.T) {
-	imageStore := image.MockStore{}
+	// In order to validate that cached data used cache "will return" some other data from what http server would
+	testImage := []byte(fmt.Sprintf("%256s", "X"))
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) {
+		return testImage, nil
+	}}
 	img := Image{
 		CacheExternal: true,
 		RemarkURL:     "https://demo.remark42.com",
@@ -207,10 +227,6 @@ func TestImage_RoutesUsingCachedImage(t *testing.T) {
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
-	// In order to validate that cached data used cache "will return" some other data from what http server would
-	testImage := []byte(fmt.Sprintf("%256s", "X"))
-	imageStore.On("Load", mock.Anything).Once().Return(testImage, nil)
-
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
 	assert.NoError(t, resp.Body.Close())
@@ -219,11 +235,12 @@ func TestImage_RoutesUsingCachedImage(t *testing.T) {
 	assert.Equal(t, "text/plain; charset=utf-8", resp.Header["Content-Type"][0],
 		"if you save text you receive text/plain in response, that's only fair option you got")
 
-	imageStore.AssertCalled(t, "Load", mock.Anything)
+	assert.Equal(t, 1, len(imageStore.LoadCalls()))
 }
 
 func TestImage_RoutesTimedOut(t *testing.T) {
-	imageStore := image.MockStore{}
+	// no image supposed to be cached
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",
@@ -239,9 +256,6 @@ func TestImage_RoutesTimedOut(t *testing.T) {
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
 
-	// no image supposed to be cached
-	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
-
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
@@ -250,6 +264,7 @@ func TestImage_RoutesTimedOut(t *testing.T) {
 	require.NoError(t, err)
 	t.Log(string(b))
 	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
+	assert.Equal(t, 1, len(imageStore.LoadCalls()))
 }
 
 func TestImage_ConvertProxyMode(t *testing.T) {

backend/app/store/admin/admin.go

@@ -8,6 +8,9 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
+//go:generate moq --out admin_mock.go . Store
+
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key(siteID string) (key string, err error)
@@ -78,5 +81,5 @@ func (s *StaticStore) Enabled(site string) (ok bool, err error) {
 	return false, nil
 }
 
-// OnEvent doesn nothing for StaticStore
+// OnEvent does nothing for StaticStore
 func (s *StaticStore) OnEvent(_ string, _ EventType) error { return nil }

backend/app/store/admin/admin_mock.go

@@ -0,0 +1,256 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package admin
+
+import (
+	"sync"
+)
+
+// Ensure, that StoreMock does implement Store.
+// If this is not the case, regenerate this file with moq.
+var _ Store = &StoreMock{}
+
+// StoreMock is a mock implementation of Store.
+//
+//	func TestSomethingThatUsesStore(t *testing.T) {
+//
+//		// make and configure a mocked Store
+//		mockedStore := &StoreMock{
+//			AdminsFunc: func(siteID string) ([]string, error) {
+//				panic("mock out the Admins method")
+//			},
+//			EmailFunc: func(siteID string) (string, error) {
+//				panic("mock out the Email method")
+//			},
+//			EnabledFunc: func(siteID string) (bool, error) {
+//				panic("mock out the Enabled method")
+//			},
+//			KeyFunc: func(siteID string) (string, error) {
+//				panic("mock out the Key method")
+//			},
+//			OnEventFunc: func(siteID string, et EventType) error {
+//				panic("mock out the OnEvent method")
+//			},
+//		}
+//
+//		// use mockedStore in code that requires Store
+//		// and then make assertions.
+//
+//	}
+type StoreMock struct {
+	// AdminsFunc mocks the Admins method.
+	AdminsFunc func(siteID string) ([]string, error)
+
+	// EmailFunc mocks the Email method.
+	EmailFunc func(siteID string) (string, error)
+
+	// EnabledFunc mocks the Enabled method.
+	EnabledFunc func(siteID string) (bool, error)
+
+	// KeyFunc mocks the Key method.
+	KeyFunc func(siteID string) (string, error)
+
+	// OnEventFunc mocks the OnEvent method.
+	OnEventFunc func(siteID string, et EventType) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Admins holds details about calls to the Admins method.
+		Admins []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Email holds details about calls to the Email method.
+		Email []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Enabled holds details about calls to the Enabled method.
+		Enabled []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Key holds details about calls to the Key method.
+		Key []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// OnEvent holds details about calls to the OnEvent method.
+		OnEvent []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+			// Et is the et argument value.
+			Et EventType
+		}
+	}
+	lockAdmins  sync.RWMutex
+	lockEmail   sync.RWMutex
+	lockEnabled sync.RWMutex
+	lockKey     sync.RWMutex
+	lockOnEvent sync.RWMutex
+}
+
+// Admins calls AdminsFunc.
+func (mock *StoreMock) Admins(siteID string) ([]string, error) {
+	if mock.AdminsFunc == nil {
+		panic("StoreMock.AdminsFunc: method is nil but Store.Admins was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockAdmins.Lock()
+	mock.calls.Admins = append(mock.calls.Admins, callInfo)
+	mock.lockAdmins.Unlock()
+	return mock.AdminsFunc(siteID)
+}
+
+// AdminsCalls gets all the calls that were made to Admins.
+// Check the length with:
+//
+//	len(mockedStore.AdminsCalls())
+func (mock *StoreMock) AdminsCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockAdmins.RLock()
+	calls = mock.calls.Admins
+	mock.lockAdmins.RUnlock()
+	return calls
+}
+
+// Email calls EmailFunc.
+func (mock *StoreMock) Email(siteID string) (string, error) {
+	if mock.EmailFunc == nil {
+		panic("StoreMock.EmailFunc: method is nil but Store.Email was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEmail.Lock()
+	mock.calls.Email = append(mock.calls.Email, callInfo)
+	mock.lockEmail.Unlock()
+	return mock.EmailFunc(siteID)
+}
+
+// EmailCalls gets all the calls that were made to Email.
+// Check the length with:
+//
+//	len(mockedStore.EmailCalls())
+func (mock *StoreMock) EmailCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEmail.RLock()
+	calls = mock.calls.Email
+	mock.lockEmail.RUnlock()
+	return calls
+}
+
+// Enabled calls EnabledFunc.
+func (mock *StoreMock) Enabled(siteID string) (bool, error) {
+	if mock.EnabledFunc == nil {
+		panic("StoreMock.EnabledFunc: method is nil but Store.Enabled was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEnabled.Lock()
+	mock.calls.Enabled = append(mock.calls.Enabled, callInfo)
+	mock.lockEnabled.Unlock()
+	return mock.EnabledFunc(siteID)
+}
+
+// EnabledCalls gets all the calls that were made to Enabled.
+// Check the length with:
+//
+//	len(mockedStore.EnabledCalls())
+func (mock *StoreMock) EnabledCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEnabled.RLock()
+	calls = mock.calls.Enabled
+	mock.lockEnabled.RUnlock()
+	return calls
+}
+
+// Key calls KeyFunc.
+func (mock *StoreMock) Key(siteID string) (string, error) {
+	if mock.KeyFunc == nil {
+		panic("StoreMock.KeyFunc: method is nil but Store.Key was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockKey.Lock()
+	mock.calls.Key = append(mock.calls.Key, callInfo)
+	mock.lockKey.Unlock()
+	return mock.KeyFunc(siteID)
+}
+
+// KeyCalls gets all the calls that were made to Key.
+// Check the length with:
+//
+//	len(mockedStore.KeyCalls())
+func (mock *StoreMock) KeyCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockKey.RLock()
+	calls = mock.calls.Key
+	mock.lockKey.RUnlock()
+	return calls
+}
+
+// OnEvent calls OnEventFunc.
+func (mock *StoreMock) OnEvent(siteID string, et EventType) error {
+	if mock.OnEventFunc == nil {
+		panic("StoreMock.OnEventFunc: method is nil but Store.OnEvent was just called")
+	}
+	callInfo := struct {
+		SiteID string
+		Et     EventType
+	}{
+		SiteID: siteID,
+		Et:     et,
+	}
+	mock.lockOnEvent.Lock()
+	mock.calls.OnEvent = append(mock.calls.OnEvent, callInfo)
+	mock.lockOnEvent.Unlock()
+	return mock.OnEventFunc(siteID, et)
+}
+
+// OnEventCalls gets all the calls that were made to OnEvent.
+// Check the length with:
+//
+//	len(mockedStore.OnEventCalls())
+func (mock *StoreMock) OnEventCalls() []struct {
+	SiteID string
+	Et     EventType
+} {
+	var calls []struct {
+		SiteID string
+		Et     EventType
+	}
+	mock.lockOnEvent.RLock()
+	calls = mock.calls.OnEvent
+	mock.lockOnEvent.RUnlock()
+	return calls
+}

backend/app/store/admin/admin_test.go

@@ -6,6 +6,20 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStaticStore_StoreWithoutSites(t *testing.T) {
+	var ks Store = NewStaticKeyStore("key123")
+	enabled, err := ks.Enabled("any")
+	assert.NoError(t, err)
+	assert.True(t, enabled, "on empty store all sites are enabled")
+	assert.NoError(t, ks.OnEvent("test", EvCreate), "static store does nothing OnEvent")
+
+	// empty key
+	ks = NewStaticKeyStore("")
+	key, err := ks.Key("any")
+	assert.Error(t, err, "empty key")
+	assert.Empty(t, key)
+}
+
 func TestStaticStore_Get(t *testing.T) {
 	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"},
 		[]string{"123", "xyz"}, "aa@example.com")

backend/app/store/comment.go

@@ -15,7 +15,7 @@ type Comment struct {
 	ID          string                 `json:"id" bson:"_id"`
 	ParentID    string                 `json:"pid"`
 	Text        string                 `json:"text"`
-	Orig        string                 `json:"orig,omitempty"`
+	Orig        string                 `json:"orig,omitempty"` // important: never render this as HTML! It's not sanitized.
 	User        User                   `json:"user"`
 	Locator     Locator                `json:"locator"`
 	Score       int                    `json:"score"`
@@ -45,9 +45,9 @@ type Edit struct {
 
 // PostInfo holds summary for given post url
 type PostInfo struct {
-	URL      string    `json:"url"`
+	URL      string    `json:"url,omitempty"` // can be attached to site-wide comments but won't be set then
 	Count    int       `json:"count"`
-	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"`
+	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"` // can be attached to site-wide comments but won't be set then
 	FirstTS  time.Time `json:"first_time,omitempty" bson:"first_time,omitempty"`
 	LastTS   time.Time `json:"last_time,omitempty" bson:"last_time,omitempty"`
 }
@@ -98,6 +98,7 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 	c.Text = ""
 	c.Orig = ""
 	c.Score = 0
+	c.Controversy = 0
 	c.Votes = map[string]bool{}
 	c.VotedIPs = make(map[string]VotedIPInfo)
 	c.Edit = nil
@@ -112,20 +113,23 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 	}
 }
 
-// Sanitize clean dangerous html/js from the comment
+// Sanitize clean dangerous html/js from the comment.
+// Comment.Orig which is used to store the original comment text is not sanitized
+// as we expect to never render it as HTML and render Comment.Text instead
 func (c *Comment) Sanitize() {
 	p := bluemonday.UGCPolicy()
 	p.AllowAttrs("class").Matching(regexp.MustCompile("^chroma$")).OnElements("pre")
+	// special case for embedding the quotes from Twitter
+	p.AllowAttrs("class").Matching(regexp.MustCompile("^twitter-tweet$")).OnElements("blockquote")
 	// this is list of <span> tag classes which could be produced by chroma code renderer
-	// source: https://github.com/alecthomas/chroma/blob/022b6f4fc2c4aa819aac18363c8de3f70619200b/types.go#L221-L316
-	const codeSpanClassRegex = "^(chroma|ln|lnt|hl|lntable|lntd|w|err|x|esc|k|kc" +
+	// source: https://github.com/alecthomas/chroma/blob/c263f6f/types.go#L209-L306
+	const codeSpanClassRegex = "^(bg|chroma|line|ln|lnt|hl|lntable|lntd|lnlinks|cl|w|err|x|k|kc" +
 		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
 		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
 		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
 	p.AllowAttrs("class").Matching(regexp.MustCompile(codeSpanClassRegex)).OnElements("span")
 	p.AllowAttrs("loading").Matching(regexp.MustCompile("^(lazy|eager)$")).OnElements("img")
 	c.Text = p.Sanitize(c.Text)
-	c.Orig = p.Sanitize(c.Orig)
 	c.User.ID = template.HTMLEscapeString(c.User.ID)
 	c.User.Name = c.SanitizeText(c.User.Name)
 	c.User.Picture = c.SanitizeAsURL(c.User.Picture)
@@ -143,7 +147,7 @@ func (c *Comment) Snippet(limit int) string {
 	if size < limit {
 		return cleanText
 	}
-	snippet := []rune(cleanText)[:size]
+	snippet := []rune(cleanText)[:limit]
 	// go back in snippet and found the first space
 	for i := len(snippet) - 1; i >= 0; i-- {
 		if snippet[i] == ' ' {
@@ -151,6 +155,10 @@ func (c *Comment) Snippet(limit int) string {
 			break
 		}
 	}
+	// Don't add a space if comment is just a one single word which has been truncated.
+	if len(snippet) == limit {
+		return string(snippet) + "..."
+	}
 	return string(snippet) + " ..."
 }
 
@@ -175,8 +183,8 @@ func (c *Comment) escapeHTMLWithSome(inp string) string {
 	return res
 }
 
-// SanitizeText used to sanitize any input string
+// SanitizeText used to sanitize any input string, and removes any HTML tags
 func (c *Comment) SanitizeText(inp string) string {
-	clean := bluemonday.UGCPolicy().Sanitize(inp)
-	return c.escapeHTMLWithSome(clean)
+	clean := bluemonday.StrictPolicy().Sanitize(inp)
+	return strings.TrimSpace(c.escapeHTMLWithSome(clean))
 }

backend/app/store/comment_test.go

@@ -22,7 +22,7 @@ func TestComment_Sanitize(t *testing.T) {
 			},
 			out: Comment{
 				Text: "blah XSS\n\t",
-				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
+				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name"},
 			},
 		},
 		{
@@ -88,6 +88,18 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah blah", PostTitle: "<script>alert()</script>something"},
 			out: Comment{Text: "blah blah", PostTitle: "something"},
 		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "<a href=\"https://example.com\">test</a>"},
+			out: Comment{Text: "blah blah", PostTitle: "test"},
+		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"}, // link is left as-is, but not rendered as <a>
+			out: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"},
+		},
+		{
+			inp: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw">@marcsheep</a> <a href="https://t.co/ULnVpG8w55">pic.twitter.com/ULnVpG8w55</a></p>&mdash; Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw">November 9, 2020</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>`, PostTitle: "Twitter quote"},
+			out: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn" rel="nofollow">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw" rel="nofollow">@marcsheep</a> <a href="https://t.co/ULnVpG8w55" rel="nofollow">pic.twitter.com/ULnVpG8w55</a></p>— Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw" rel="nofollow">November 9, 2020</a></blockquote> `, PostTitle: "Twitter quote"},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -192,8 +204,8 @@ func TestComment_Snippet(t *testing.T) {
 		{0, "", ""},
 		{-1, "test\nblah", "test blah"},
 		{5, "test\nblah", "test ..."},
-		{5, "xyz12345 xxx", "xyz12345 ..."},
-		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+		{5, "xyz12345 xxx", "xyz12..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 ..."},
 	}
 
 	for i, tt := range tbl {
@@ -259,7 +271,7 @@ func TestComment_sanitizeText(t *testing.T) {
 		},
 		{
 			"<a href=javascript:alert(document.domain)//>xxx</a>",
-			"&lt;a/&gt;xxx&lt;/a&gt;",
+			"xxx",
 		},
 	}
 

backend/app/store/engine/bolt.go

@@ -16,15 +16,15 @@ import (
 
 // BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
 // there are 6 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - users details in "user_details" bucket. Key is userID, value - UserDetailEntry
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
+//   - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
+//   - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
+//     value is not full comment but a reference combined from post-url+commentID
+//   - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
+//     is a nested bucket named userID with kv as ts:reference
+//   - users details in "user_details" bucket. Key is userID, value - UserDetailEntry
+//   - blocking info sits in "block" bucket. Key is userID, value - ts
+//   - counts per post to keep number of comments. Key is post url, value - count
+//   - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
 type BoltDB struct {
 	dbs map[string]*bolt.DB
 }
@@ -170,7 +170,7 @@ func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
 				return e
 			}
 
-			return bucket.ForEach(func(k, v []byte) error {
+			return bucket.ForEach(func(_, v []byte) error {
 				comment := store.Comment{}
 				if e = json.Unmarshal(v, &comment); e != nil {
 					return fmt.Errorf("failed to unmarshal: %w", e)
@@ -306,9 +306,8 @@ func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
 		})
 
 		// set read-only from age and manual bucket
-		readOnlyAge := req.ReadOnlyAge
-		info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-		if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly && b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
 			info.ReadOnly = true
 		}
 		return []store.PostInfo{info}, err
@@ -425,11 +424,11 @@ func (b *BoltDB) Close() error {
 }
 
 // Last returns up to max last comments for given siteID
-func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+func (b *BoltDB) lastComments(siteID string, maximum int, since time.Time) (comments []store.Comment, err error) {
 	comments = []store.Comment{}
 
-	if max > lastLimit || max == 0 {
-		max = lastLimit
+	if maximum > lastLimit || maximum == 0 {
+		maximum = lastLimit
 	}
 
 	bdb, err := b.db(siteID)
@@ -467,7 +466,7 @@ func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments
 				continue
 			}
 			comments = append(comments, comment)
-			if len(comments) >= max {
+			if len(comments) >= maximum {
 				break
 			}
 		}
@@ -725,7 +724,7 @@ func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err e
 	err = bdb.View(func(tx *bolt.Tx) error {
 		var entry UserDetailEntry
 		bucket := tx.Bucket([]byte(userDetailsBucketName))
-		return bucket.ForEach(func(userID, value []byte) error {
+		return bucket.ForEach(func(_, value []byte) error {
 			if err = json.Unmarshal(value, &entry); err != nil {
 				return fmt.Errorf("failed to unmarshal entry: %w", e)
 			}
@@ -780,7 +779,7 @@ func (b *BoltDB) deleteUserDetail(bdb *bolt.DB, userID string, userDetail UserDe
 	}
 
 	return bdb.Update(func(tx *bolt.Tx) error {
-		// updated entry is not empty and we need to store it's updated copy
+		// updated entry is not empty and we need to store its updated copy
 		err := b.save(tx.Bucket([]byte(userDetailsBucketName)), userID, entry)
 		if err != nil {
 			return fmt.Errorf("failed to update detail %s for %s: %w", userDetail, userID, err)
@@ -870,7 +869,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID, userID string, mode store.Dele
 		err = bdb.View(func(tx *bolt.Tx) error {
 			postsBkt := tx.Bucket([]byte(postsBucketName))
 			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+			err = postBkt.ForEach(func(_ []byte, commentVal []byte) error {
 				comment := store.Comment{}
 				if err = json.Unmarshal(commentVal, &comment); err != nil {
 					return fmt.Errorf("failed to unmarshal: %w", err)

backend/app/store/engine/bolt_test.go

@@ -911,6 +911,13 @@ func TestBoltDB_NewFailed(t *testing.T) {
 	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
 }
 
+func TestBoltDB_DoubleClose(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+	assert.NoError(t, b.Close())
+	assert.NoError(t, b.Close(), "second call should not result in panic or errors")
+}
+
 // makes new boltdb, put two records
 func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	_ = os.Remove(testDB)

backend/app/store/engine/engine.go

@@ -11,8 +11,8 @@ import (
 	"github.com/umputun/remark42/backend/app/store"
 )
 
-// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
-//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
+// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
+//go:generate moq --out engine_mock.go . Interface
 
 // Interface defines methods provided by low-level storage engine
 type Interface interface {

backend/app/store/engine/engine_mock.go

@@ -1,231 +1,508 @@
-// Code generated by mockery v1.1.2. DO NOT EDIT.
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package engine
 
 import (
-	mock "github.com/stretchr/testify/mock"
 	store "github.com/umputun/remark42/backend/app/store"
+	"sync"
 )
 
-// MockInterface is an autogenerated mock type for the Interface type
-type MockInterface struct {
-	mock.Mock
-}
+// Ensure, that InterfaceMock does implement Interface.
+// If this is not the case, regenerate this file with moq.
+var _ Interface = &InterfaceMock{}
 
-// Close provides a mock function with given fields:
-func (_m *MockInterface) Close() error {
-	ret := _m.Called()
+// InterfaceMock is a mock implementation of Interface.
+//
+//	func TestSomethingThatUsesInterface(t *testing.T) {
+//
+//		// make and configure a mocked Interface
+//		mockedInterface := &InterfaceMock{
+//			CloseFunc: func() error {
+//				panic("mock out the Close method")
+//			},
+//			CountFunc: func(req FindRequest) (int, error) {
+//				panic("mock out the Count method")
+//			},
+//			CreateFunc: func(comment store.Comment) (string, error) {
+//				panic("mock out the Create method")
+//			},
+//			DeleteFunc: func(req DeleteRequest) error {
+//				panic("mock out the Delete method")
+//			},
+//			FindFunc: func(req FindRequest) ([]store.Comment, error) {
+//				panic("mock out the Find method")
+//			},
+//			FlagFunc: func(req FlagRequest) (bool, error) {
+//				panic("mock out the Flag method")
+//			},
+//			GetFunc: func(req GetRequest) (store.Comment, error) {
+//				panic("mock out the Get method")
+//			},
+//			InfoFunc: func(req InfoRequest) ([]store.PostInfo, error) {
+//				panic("mock out the Info method")
+//			},
+//			ListFlagsFunc: func(req FlagRequest) ([]interface{}, error) {
+//				panic("mock out the ListFlags method")
+//			},
+//			UpdateFunc: func(comment store.Comment) error {
+//				panic("mock out the Update method")
+//			},
+//			UserDetailFunc: func(req UserDetailRequest) ([]UserDetailEntry, error) {
+//				panic("mock out the UserDetail method")
+//			},
+//		}
+//
+//		// use mockedInterface in code that requires Interface
+//		// and then make assertions.
+//
+//	}
+type InterfaceMock struct {
+	// CloseFunc mocks the Close method.
+	CloseFunc func() error
 
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
+	// CountFunc mocks the Count method.
+	CountFunc func(req FindRequest) (int, error)
 
-	return r0
-}
+	// CreateFunc mocks the Create method.
+	CreateFunc func(comment store.Comment) (string, error)
 
-// Count provides a mock function with given fields: req
-func (_m *MockInterface) Count(req FindRequest) (int, error) {
-	ret := _m.Called(req)
+	// DeleteFunc mocks the Delete method.
+	DeleteFunc func(req DeleteRequest) error
 
-	var r0 int
-	if rf, ok := ret.Get(0).(func(FindRequest) int); ok {
-		r0 = rf(req)
-	} else {
-		r0 = ret.Get(0).(int)
-	}
+	// FindFunc mocks the Find method.
+	FindFunc func(req FindRequest) ([]store.Comment, error)
 
-	var r1 error
-	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
-	}
+	// FlagFunc mocks the Flag method.
+	FlagFunc func(req FlagRequest) (bool, error)
 
-	return r0, r1
-}
+	// GetFunc mocks the Get method.
+	GetFunc func(req GetRequest) (store.Comment, error)
 
-// Create provides a mock function with given fields: comment
-func (_m *MockInterface) Create(comment store.Comment) (string, error) {
-	ret := _m.Called(comment)
+	// InfoFunc mocks the Info method.
+	InfoFunc func(req InfoRequest) ([]store.PostInfo, error)
 
-	var r0 string
-	if rf, ok := ret.Get(0).(func(store.Comment) string); ok {
-		r0 = rf(comment)
-	} else {
-		r0 = ret.Get(0).(string)
-	}
+	// ListFlagsFunc mocks the ListFlags method.
+	ListFlagsFunc func(req FlagRequest) ([]interface{}, error)
 
-	var r1 error
-	if rf, ok := ret.Get(1).(func(store.Comment) error); ok {
-		r1 = rf(comment)
-	} else {
-		r1 = ret.Error(1)
-	}
+	// UpdateFunc mocks the Update method.
+	UpdateFunc func(comment store.Comment) error
 
-	return r0, r1
-}
+	// UserDetailFunc mocks the UserDetail method.
+	UserDetailFunc func(req UserDetailRequest) ([]UserDetailEntry, error)
 
-// Delete provides a mock function with given fields: req
-func (_m *MockInterface) Delete(req DeleteRequest) error {
-	ret := _m.Called(req)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok {
-		r0 = rf(req)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Find provides a mock function with given fields: req
-func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) {
-	ret := _m.Called(req)
-
-	var r0 []store.Comment
-	if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok {
-		r0 = rf(req)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]store.Comment)
+	// calls tracks calls to the methods.
+	calls struct {
+		// Close holds details about calls to the Close method.
+		Close []struct {
+		}
+		// Count holds details about calls to the Count method.
+		Count []struct {
+			// Req is the req argument value.
+			Req FindRequest
+		}
+		// Create holds details about calls to the Create method.
+		Create []struct {
+			// Comment is the comment argument value.
+			Comment store.Comment
+		}
+		// Delete holds details about calls to the Delete method.
+		Delete []struct {
+			// Req is the req argument value.
+			Req DeleteRequest
+		}
+		// Find holds details about calls to the Find method.
+		Find []struct {
+			// Req is the req argument value.
+			Req FindRequest
+		}
+		// Flag holds details about calls to the Flag method.
+		Flag []struct {
+			// Req is the req argument value.
+			Req FlagRequest
+		}
+		// Get holds details about calls to the Get method.
+		Get []struct {
+			// Req is the req argument value.
+			Req GetRequest
+		}
+		// Info holds details about calls to the Info method.
+		Info []struct {
+			// Req is the req argument value.
+			Req InfoRequest
+		}
+		// ListFlags holds details about calls to the ListFlags method.
+		ListFlags []struct {
+			// Req is the req argument value.
+			Req FlagRequest
+		}
+		// Update holds details about calls to the Update method.
+		Update []struct {
+			// Comment is the comment argument value.
+			Comment store.Comment
+		}
+		// UserDetail holds details about calls to the UserDetail method.
+		UserDetail []struct {
+			// Req is the req argument value.
+			Req UserDetailRequest
 		}
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	lockClose      sync.RWMutex
+	lockCount      sync.RWMutex
+	lockCreate     sync.RWMutex
+	lockDelete     sync.RWMutex
+	lockFind       sync.RWMutex
+	lockFlag       sync.RWMutex
+	lockGet        sync.RWMutex
+	lockInfo       sync.RWMutex
+	lockListFlags  sync.RWMutex
+	lockUpdate     sync.RWMutex
+	lockUserDetail sync.RWMutex
 }
 
-// Flag provides a mock function with given fields: req
-func (_m *MockInterface) Flag(req FlagRequest) (bool, error) {
-	ret := _m.Called(req)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok {
-		r0 = rf(req)
-	} else {
-		r0 = ret.Get(0).(bool)
+// Close calls CloseFunc.
+func (mock *InterfaceMock) Close() error {
+	if mock.CloseFunc == nil {
+		panic("InterfaceMock.CloseFunc: method is nil but Interface.Close was just called")
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	callInfo := struct {
+	}{}
+	mock.lockClose.Lock()
+	mock.calls.Close = append(mock.calls.Close, callInfo)
+	mock.lockClose.Unlock()
+	return mock.CloseFunc()
 }
 
-// Get provides a mock function with given fields: req
-func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) {
-	ret := _m.Called(req)
-
-	var r0 store.Comment
-	if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok {
-		r0 = rf(req)
-	} else {
-		r0 = ret.Get(0).(store.Comment)
+// CloseCalls gets all the calls that were made to Close.
+// Check the length with:
+//
+//	len(mockedInterface.CloseCalls())
+func (mock *InterfaceMock) CloseCalls() []struct {
+} {
+	var calls []struct {
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(GetRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	mock.lockClose.RLock()
+	calls = mock.calls.Close
+	mock.lockClose.RUnlock()
+	return calls
 }
 
-// Info provides a mock function with given fields: req
-func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) {
-	ret := _m.Called(req)
-
-	var r0 []store.PostInfo
-	if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok {
-		r0 = rf(req)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]store.PostInfo)
-		}
+// Count calls CountFunc.
+func (mock *InterfaceMock) Count(req FindRequest) (int, error) {
+	if mock.CountFunc == nil {
+		panic("InterfaceMock.CountFunc: method is nil but Interface.Count was just called")
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(InfoRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
+	callInfo := struct {
+		Req FindRequest
+	}{
+		Req: req,
 	}
-
-	return r0, r1
+	mock.lockCount.Lock()
+	mock.calls.Count = append(mock.calls.Count, callInfo)
+	mock.lockCount.Unlock()
+	return mock.CountFunc(req)
 }
 
-// ListFlags provides a mock function with given fields: req
-func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) {
-	ret := _m.Called(req)
-
-	var r0 []interface{}
-	if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok {
-		r0 = rf(req)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]interface{})
-		}
+// CountCalls gets all the calls that were made to Count.
+// Check the length with:
+//
+//	len(mockedInterface.CountCalls())
+func (mock *InterfaceMock) CountCalls() []struct {
+	Req FindRequest
+} {
+	var calls []struct {
+		Req FindRequest
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	mock.lockCount.RLock()
+	calls = mock.calls.Count
+	mock.lockCount.RUnlock()
+	return calls
 }
 
-// Update provides a mock function with given fields: comment
-func (_m *MockInterface) Update(comment store.Comment) error {
-	ret := _m.Called(comment)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(store.Comment) error); ok {
-		r0 = rf(comment)
-	} else {
-		r0 = ret.Error(0)
+// Create calls CreateFunc.
+func (mock *InterfaceMock) Create(comment store.Comment) (string, error) {
+	if mock.CreateFunc == nil {
+		panic("InterfaceMock.CreateFunc: method is nil but Interface.Create was just called")
 	}
-
-	return r0
+	callInfo := struct {
+		Comment store.Comment
+	}{
+		Comment: comment,
+	}
+	mock.lockCreate.Lock()
+	mock.calls.Create = append(mock.calls.Create, callInfo)
+	mock.lockCreate.Unlock()
+	return mock.CreateFunc(comment)
 }
 
-// UserDetail provides a mock function with given fields: req
-func (_m *MockInterface) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
-	ret := _m.Called(req)
-
-	var r0 []UserDetailEntry
-	if rf, ok := ret.Get(0).(func(UserDetailRequest) []UserDetailEntry); ok {
-		r0 = rf(req)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]UserDetailEntry)
-		}
+// CreateCalls gets all the calls that were made to Create.
+// Check the length with:
+//
+//	len(mockedInterface.CreateCalls())
+func (mock *InterfaceMock) CreateCalls() []struct {
+	Comment store.Comment
+} {
+	var calls []struct {
+		Comment store.Comment
 	}
+	mock.lockCreate.RLock()
+	calls = mock.calls.Create
+	mock.lockCreate.RUnlock()
+	return calls
+}
 
-	var r1 error
-	if rf, ok := ret.Get(1).(func(UserDetailRequest) error); ok {
-		r1 = rf(req)
-	} else {
-		r1 = ret.Error(1)
+// Delete calls DeleteFunc.
+func (mock *InterfaceMock) Delete(req DeleteRequest) error {
+	if mock.DeleteFunc == nil {
+		panic("InterfaceMock.DeleteFunc: method is nil but Interface.Delete was just called")
 	}
+	callInfo := struct {
+		Req DeleteRequest
+	}{
+		Req: req,
+	}
+	mock.lockDelete.Lock()
+	mock.calls.Delete = append(mock.calls.Delete, callInfo)
+	mock.lockDelete.Unlock()
+	return mock.DeleteFunc(req)
+}
+
+// DeleteCalls gets all the calls that were made to Delete.
+// Check the length with:
+//
+//	len(mockedInterface.DeleteCalls())
+func (mock *InterfaceMock) DeleteCalls() []struct {
+	Req DeleteRequest
+} {
+	var calls []struct {
+		Req DeleteRequest
+	}
+	mock.lockDelete.RLock()
+	calls = mock.calls.Delete
+	mock.lockDelete.RUnlock()
+	return calls
+}
+
+// Find calls FindFunc.
+func (mock *InterfaceMock) Find(req FindRequest) ([]store.Comment, error) {
+	if mock.FindFunc == nil {
+		panic("InterfaceMock.FindFunc: method is nil but Interface.Find was just called")
+	}
+	callInfo := struct {
+		Req FindRequest
+	}{
+		Req: req,
+	}
+	mock.lockFind.Lock()
+	mock.calls.Find = append(mock.calls.Find, callInfo)
+	mock.lockFind.Unlock()
+	return mock.FindFunc(req)
+}
+
+// FindCalls gets all the calls that were made to Find.
+// Check the length with:
+//
+//	len(mockedInterface.FindCalls())
+func (mock *InterfaceMock) FindCalls() []struct {
+	Req FindRequest
+} {
+	var calls []struct {
+		Req FindRequest
+	}
+	mock.lockFind.RLock()
+	calls = mock.calls.Find
+	mock.lockFind.RUnlock()
+	return calls
+}
 
-	return r0, r1
+// Flag calls FlagFunc.
+func (mock *InterfaceMock) Flag(req FlagRequest) (bool, error) {
+	if mock.FlagFunc == nil {
+		panic("InterfaceMock.FlagFunc: method is nil but Interface.Flag was just called")
+	}
+	callInfo := struct {
+		Req FlagRequest
+	}{
+		Req: req,
+	}
+	mock.lockFlag.Lock()
+	mock.calls.Flag = append(mock.calls.Flag, callInfo)
+	mock.lockFlag.Unlock()
+	return mock.FlagFunc(req)
+}
+
+// FlagCalls gets all the calls that were made to Flag.
+// Check the length with:
+//
+//	len(mockedInterface.FlagCalls())
+func (mock *InterfaceMock) FlagCalls() []struct {
+	Req FlagRequest
+} {
+	var calls []struct {
+		Req FlagRequest
+	}
+	mock.lockFlag.RLock()
+	calls = mock.calls.Flag
+	mock.lockFlag.RUnlock()
+	return calls
+}
+
+// Get calls GetFunc.
+func (mock *InterfaceMock) Get(req GetRequest) (store.Comment, error) {
+	if mock.GetFunc == nil {
+		panic("InterfaceMock.GetFunc: method is nil but Interface.Get was just called")
+	}
+	callInfo := struct {
+		Req GetRequest
+	}{
+		Req: req,
+	}
+	mock.lockGet.Lock()
+	mock.calls.Get = append(mock.calls.Get, callInfo)
+	mock.lockGet.Unlock()
+	return mock.GetFunc(req)
+}
+
+// GetCalls gets all the calls that were made to Get.
+// Check the length with:
+//
+//	len(mockedInterface.GetCalls())
+func (mock *InterfaceMock) GetCalls() []struct {
+	Req GetRequest
+} {
+	var calls []struct {
+		Req GetRequest
+	}
+	mock.lockGet.RLock()
+	calls = mock.calls.Get
+	mock.lockGet.RUnlock()
+	return calls
+}
+
+// Info calls InfoFunc.
+func (mock *InterfaceMock) Info(req InfoRequest) ([]store.PostInfo, error) {
+	if mock.InfoFunc == nil {
+		panic("InterfaceMock.InfoFunc: method is nil but Interface.Info was just called")
+	}
+	callInfo := struct {
+		Req InfoRequest
+	}{
+		Req: req,
+	}
+	mock.lockInfo.Lock()
+	mock.calls.Info = append(mock.calls.Info, callInfo)
+	mock.lockInfo.Unlock()
+	return mock.InfoFunc(req)
+}
+
+// InfoCalls gets all the calls that were made to Info.
+// Check the length with:
+//
+//	len(mockedInterface.InfoCalls())
+func (mock *InterfaceMock) InfoCalls() []struct {
+	Req InfoRequest
+} {
+	var calls []struct {
+		Req InfoRequest
+	}
+	mock.lockInfo.RLock()
+	calls = mock.calls.Info
+	mock.lockInfo.RUnlock()
+	return calls
+}
+
+// ListFlags calls ListFlagsFunc.
+func (mock *InterfaceMock) ListFlags(req FlagRequest) ([]interface{}, error) {
+	if mock.ListFlagsFunc == nil {
+		panic("InterfaceMock.ListFlagsFunc: method is nil but Interface.ListFlags was just called")
+	}
+	callInfo := struct {
+		Req FlagRequest
+	}{
+		Req: req,
+	}
+	mock.lockListFlags.Lock()
+	mock.calls.ListFlags = append(mock.calls.ListFlags, callInfo)
+	mock.lockListFlags.Unlock()
+	return mock.ListFlagsFunc(req)
+}
+
+// ListFlagsCalls gets all the calls that were made to ListFlags.
+// Check the length with:
+//
+//	len(mockedInterface.ListFlagsCalls())
+func (mock *InterfaceMock) ListFlagsCalls() []struct {
+	Req FlagRequest
+} {
+	var calls []struct {
+		Req FlagRequest
+	}
+	mock.lockListFlags.RLock()
+	calls = mock.calls.ListFlags
+	mock.lockListFlags.RUnlock()
+	return calls
+}
+
+// Update calls UpdateFunc.
+func (mock *InterfaceMock) Update(comment store.Comment) error {
+	if mock.UpdateFunc == nil {
+		panic("InterfaceMock.UpdateFunc: method is nil but Interface.Update was just called")
+	}
+	callInfo := struct {
+		Comment store.Comment
+	}{
+		Comment: comment,
+	}
+	mock.lockUpdate.Lock()
+	mock.calls.Update = append(mock.calls.Update, callInfo)
+	mock.lockUpdate.Unlock()
+	return mock.UpdateFunc(comment)
+}
+
+// UpdateCalls gets all the calls that were made to Update.
+// Check the length with:
+//
+//	len(mockedInterface.UpdateCalls())
+func (mock *InterfaceMock) UpdateCalls() []struct {
+	Comment store.Comment
+} {
+	var calls []struct {
+		Comment store.Comment
+	}
+	mock.lockUpdate.RLock()
+	calls = mock.calls.Update
+	mock.lockUpdate.RUnlock()
+	return calls
+}
+
+// UserDetail calls UserDetailFunc.
+func (mock *InterfaceMock) UserDetail(req UserDetailRequest) ([]UserDetailEntry, error) {
+	if mock.UserDetailFunc == nil {
+		panic("InterfaceMock.UserDetailFunc: method is nil but Interface.UserDetail was just called")
+	}
+	callInfo := struct {
+		Req UserDetailRequest
+	}{
+		Req: req,
+	}
+	mock.lockUserDetail.Lock()
+	mock.calls.UserDetail = append(mock.calls.UserDetail, callInfo)
+	mock.lockUserDetail.Unlock()
+	return mock.UserDetailFunc(req)
+}
+
+// UserDetailCalls gets all the calls that were made to UserDetail.
+// Check the length with:
+//
+//	len(mockedInterface.UserDetailCalls())
+func (mock *InterfaceMock) UserDetailCalls() []struct {
+	Req UserDetailRequest
+} {
+	var calls []struct {
+		Req UserDetailRequest
+	}
+	mock.lockUserDetail.RLock()
+	calls = mock.calls.UserDetail
+	mock.lockUserDetail.RUnlock()
+	return calls
 }

backend/app/store/engine/remote.go

@@ -71,14 +71,44 @@ func (r *RPC) Flag(req FlagRequest) (status bool, err error) {
 	return status, err
 }
 
+func unmarshalString(data []byte) ([]interface{}, error) {
+	var strings []string
+	if err := json.Unmarshal(data, &strings); err != nil {
+		return nil, err
+	}
+	list := make([]interface{}, 0, len(strings))
+	for _, w := range strings {
+		list = append(list, w)
+	}
+	return list, nil
+}
+
+func unmarshalBlockedUser(data []byte) ([]interface{}, error) {
+	var blockedUsers []store.BlockedUser
+	if err := json.Unmarshal(data, &blockedUsers); err != nil {
+		return nil, err
+	}
+	list := make([]interface{}, 0, len(blockedUsers))
+	for _, w := range blockedUsers {
+		list = append(list, w)
+	}
+	return list, nil
+}
+
 // ListFlags get list of flagged keys, like blocked & verified user
-func (r *RPC) ListFlags(req FlagRequest) (list []interface{}, err error) {
+func (r *RPC) ListFlags(req FlagRequest) ([]interface{}, error) {
 	resp, err := r.Call("store.list_flags", req)
 	if err != nil {
 		return nil, err
 	}
-	err = json.Unmarshal(*resp.Result, &list)
-	return list, err
+	// try []store.BlockedUser
+	list, err := unmarshalBlockedUser(*resp.Result)
+	if err == nil {
+		return list, nil
+	}
+
+	// try []strings
+	return unmarshalString(*resp.Result)
 }
 
 // UserDetail sets or gets single detail value, or gets all details for requested site.

backend/app/store/engine/remote_test.go

@@ -134,7 +134,13 @@ func TestRemote_ListFlag(t *testing.T) {
 	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
 	res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked})
 	assert.NoError(t, err)
-	assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
+	require.Equal(t, 2, len(res))
+	blockedUsers := make([]store.BlockedUser, 0, len(res))
+	for _, v := range res {
+		blockedUsers = append(blockedUsers, v.(store.BlockedUser))
+	}
+	assert.Equal(t, 2, len(blockedUsers))
+	assert.Equal(t, []store.BlockedUser{{ID: "id1"}, {ID: "id2"}}, blockedUsers)
 }
 
 func TestRemote_UserDetail(t *testing.T) {
@@ -180,11 +186,17 @@ func TestRemote_Delete(t *testing.T) {
 }
 
 func TestRemote_Close(t *testing.T) {
-	ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Contains(t, string(body), "{\"method\":\"store.close\",\"id\":")
+		t.Logf("req: %s", string(body))
+		_, _ = fmt.Fprint(w, "{}")
+	}))
 	defer ts.Close()
 	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
-	err := c.Close()
-	assert.NoError(t, err)
+	assert.NoError(t, c.Close())
+	assert.NoError(t, c.Close(), "second call should not result in panic or errors")
 }
 
 func testServer(t *testing.T, req, resp string) *httptest.Server {

backend/app/store/formatter.go

@@ -4,9 +4,9 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/Depado/bfchroma"
+	"github.com/Depado/bfchroma/v2"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/v2/formatters/html"
 	bf "github.com/russross/blackfriday/v2"
 )
 
@@ -35,24 +35,17 @@ func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter {
 }
 
 // Format comment fields
-func (f *CommentFormatter) Format(c Comment) Comment {
-	c.Text = f.FormatText(c.Text)
+func (f *CommentFormatter) Format(c Comment, raw bool) Comment {
+	c.Text = f.FormatText(c.Text, raw)
 	return c
 }
 
 // FormatText converts text with markdown processor, applies external converters and shortens links
-func (f *CommentFormatter) FormatText(txt string) (res string) {
-	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
-		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
-		bf.BackslashLineBreak | bf.Autolink
-
-	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{
-		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
-	})
-
-	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
-
-	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(extRend)))
+//
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
+func (f *CommentFormatter) FormatText(txt string, raw bool) (res string) {
+	mdExt, rend := GetMdExtensionsAndRenderer(raw)
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
 	res = f.unEscape(res)
 
 	for _, conv := range f.converters {
@@ -64,14 +57,14 @@ func (f *CommentFormatter) FormatText(txt string) (res string) {
 }
 
 // Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
-func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
+func (f *CommentFormatter) shortenAutoLinks(commentHTML string, maximum int) (resHTML string) {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("a").Each(func(i int, s *goquery.Selection) {
+	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
 		if href, ok := s.Attr("href"); ok {
-			if href != s.Text() || len(href) < max+3 || max < 3 {
+			if href != s.Text() || len(href) < maximum+3 || maximum < 3 {
 				return
 			}
 			commentURL, e := url.Parse(href)
@@ -84,7 +77,7 @@ func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTM
 				return
 			}
 
-			short := string([]rune(href)[:max-3])
+			short := string([]rune(href)[:maximum-3])
 			if len(short) < len(host) {
 				short = host
 			}
@@ -117,7 +110,7 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		s.SetAttr("loading", "lazy")
 	})
 	resHTML, err = doc.Find("body").Html()
@@ -126,3 +119,23 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	}
 	return resHTML
 }
+
+// GetMdExtensionsAndRenderer returns blackfriday extensions and renderer used for rendering markdown
+// within store module.
+//
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
+func GetMdExtensionsAndRenderer(raw bool) (bf.Extensions, *bfchroma.Renderer) {
+	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
+		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
+		bf.BackslashLineBreak | bf.Autolink
+
+	flags := bf.HTMLFlags(0)
+	if !raw {
+		flags = bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes
+	}
+
+	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{Flags: flags})
+
+	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
+	return mdExt, extRend
+}

backend/app/store/formatter_test.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -35,30 +36,38 @@ func TestFormatter_FormatText(t *testing.T) {
 			"lazy image",
 		},
 		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
+		{`no_smartpants "quoted" text`, "<p>no_smartpants &#34;quoted&#34; text</p>\n!converted", "normal quotes without smartpants"},
+		{`"quoted" text`, "<p>«quoted» text</p>\n!converted", "normal quotes with smartpants"},
+		{`no_smartpants “quoted” text`, "<p>no_smartpants “quoted” text</p>\n!converted", "curly quotes without smartpants"},
+		{`“quoted” text`, "<p>“quoted” text</p>\n!converted", "curly quotes with smartpants"},
+		{`no_smartpants «quoted» text`, "<p>no_smartpants «quoted» text</p>\n!converted", "French guillemets without smartpants"},
+		{`«quoted» text`, "<p>«quoted» text</p>\n!converted", "French guillemets with smartpants"},
 		{"smth\n```go\nfunc main(aa string) int {return 0}\n```", `<p>smth</p>
-<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
 </span></span></code></pre>!converted`, "code with language"},
-		{"```\ntest_code\n```", `<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl">test_code
+		{"```\ntest_code\n```", `<pre class="chroma"><code><span class="line"><span class="cl">test_code
 </span></span></code></pre>!converted`, "code without language"},
 	}
 	f := NewCommentFormatter(mockConverter{})
 	for _, tt := range tbl {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.out, f.FormatText(tt.in))
+			raw := strings.HasPrefix(tt.in, `no_smartpants`)
+			t.Logf("raw: %v", raw)
+			assert.Equal(t, tt.out, f.FormatText(tt.in, raw))
 		})
 	}
 }
 
 func TestFormatter_FormatTextNoConverter(t *testing.T) {
 	f := NewCommentFormatter()
-	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatTextConverterFunc(t *testing.T) {
 	fn := CommentConverterFunc(func(text string) string { return "zz!" + text })
 	f := NewCommentFormatter(fn)
-	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatComment(t *testing.T) {
@@ -78,7 +87,7 @@ func TestFormatter_FormatComment(t *testing.T) {
 	f := NewCommentFormatter(mockConverter{})
 	exp := comment
 	exp.Text = "<p>blah</p>\n\n<p>xyz</p>\n!converted"
-	assert.Equal(t, exp, f.Format(comment))
+	assert.Equal(t, exp, f.Format(comment, false))
 }
 
 func TestFormatter_ShortenAutoLinks(t *testing.T) {

backend/app/store/image/bolt_store.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"time"
 
@@ -118,6 +119,18 @@ func (b *Bolt) Load(id string) ([]byte, error) {
 	return data, nil
 }
 
+// Delete image from storage
+func (b *Bolt) Delete(id string) error {
+	return b.db.Update(func(tx *bolt.Tx) error {
+		// deleting a non-existing key doesn't return an error, so joining errors from deleting an image
+		// from both buckets is safe and will return nil if there are no errors on the real delete
+		// or image is absent in both buckets
+		err := tx.Bucket([]byte(imagesBktName)).Delete([]byte(id))
+		err = errors.Join(err, tx.Bucket([]byte(imagesStagedBktName)).Delete([]byte(id)))
+		return err
+	})
+}
+
 // Cleanup runs scan of staging and removes old data based on ttl
 func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error {
 	return b.db.Update(func(tx *bolt.Tx) error {

backend/app/store/image/bolt_store_test.go

@@ -2,7 +2,6 @@ package image
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"path"
 	"testing"
@@ -59,6 +58,36 @@ func TestBoltStore_LoadAfterSave(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestBoltStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	// delete image from permanent storage
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// delete staging image
+	id = "staging_img"
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestBoltStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()
@@ -150,7 +179,7 @@ func checkBoltImgData(t *testing.T, db *bolt.DB, bucket, id string, callback fun
 }
 
 func prepareBoltImageStorageTest(t *testing.T) (svc *Bolt, teardown func()) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
+	loc, err := os.MkdirTemp("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
 
 	svc, err = NewBoltStorage(path.Join(loc, "picture.db"), bolt.Options{})

backend/app/store/image/fs_store.go

@@ -23,7 +23,8 @@ type FileSystem struct {
 	Staging    string
 	Partitions int
 
-	crc struct {
+	moveLock sync.Mutex // needed only for deleting images or moving them from staging to permanent storage
+	crc      struct {
 		*crc64.Table
 		sync.Once
 		mask    string
@@ -50,6 +51,8 @@ func (f *FileSystem) Save(id string, img []byte) error {
 
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 	log.Printf("[DEBUG] Commit image %s", id)
 	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
 
@@ -107,11 +110,31 @@ func (f *FileSystem) Load(id string) ([]byte, error) {
 	return io.ReadAll(fh)
 }
 
+// Delete image from storage
+func (f *FileSystem) Delete(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
+	staging := f.location(f.Staging, id)
+	// file doesn't exist on staging, delete from permanent location
+	if _, err := os.Stat(staging); os.IsNotExist(err) {
+		file := f.location(f.Location, id)
+		e := os.Remove(file)
+		_ = os.Remove(path.Dir(file)) // try to remove directory
+		return e
+	}
+	// delete file from staging
+	err := os.Remove(staging)
+	_ = os.Remove(path.Dir(staging)) // try to remove directory
+	return err
+}
+
 // Cleanup runs scan of staging and removes old files based on ttl
 func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error {
 	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
 		return nil
 	}
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 
 	// we can ignore context as on local FS remove is relatively fast operation
 	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
@@ -143,7 +166,7 @@ func (f *FileSystem) Info() (StoreInfo, error) {
 	}
 
 	var ts time.Time
-	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+	err := filepath.Walk(f.Staging, func(_ string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}

backend/app/store/image/fs_store_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path"
@@ -132,6 +131,31 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestFsStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// create file on staging
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestFsStore_location(t *testing.T) {
 	tbl := []struct {
 		partitions int
@@ -181,7 +205,7 @@ func TestFsStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	save := func(file string, user string) (filePath string) {
+	save := func(file, user string) (filePath string) {
 		id := path.Join(user, file)
 		err := svc.Save(id, gopherPNGBytes())
 		require.NoError(t, err)
@@ -228,7 +252,7 @@ func TestFsStore_Cleanup(t *testing.T) {
 	_, err = os.Stat(img2)
 	assert.Error(t, err, "no file on staging anymore")
 	_, err = os.Stat(img3)
-	assert.NoError(t, err, "third image is still on staging because it's cleanup timer was reset")
+	assert.NoError(t, err, "third image is still on staging because its cleanup timer was reset")
 
 	err = svc.ResetCleanupTimer("unknown_image.png")
 	assert.Error(t, err)
@@ -254,10 +278,10 @@ func TestFsStore_Info(t *testing.T) {
 }
 
 func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
+	loc, err := os.MkdirTemp("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
 
-	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	staging, err := os.MkdirTemp("", "test_image_r42.staging")
 	require.NoError(t, err, "failed to make temp staging dir")
 
 	svc = &FileSystem{

backend/app/store/image/image.go

@@ -3,7 +3,8 @@
 // Service object encloses Store and add common methods, this is the one consumer should use.
 package image
 
-//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
+// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
+//go:generate moq --out image_mock.go . Store
 
 import (
 	"bytes"
@@ -61,9 +62,6 @@ type StoreInfo struct {
 	FirstStagingImageTS time.Time
 }
 
-// To regenerate mock run from this directory:
-// sh -c "mockery -inpkg -name Store -print > /tmp/image-mock.tmp && mv /tmp/image-mock.tmp image_mock.go"
-
 // Store defines interface for saving and loading pictures.
 // Declares two-stage save with Commit. Save stores to staging area and Commit moves to the final location.
 // Two-stage commit scheme is used for not storing images which are uploaded but later never used in the comments,
@@ -72,6 +70,7 @@ type Store interface {
 	Info() (StoreInfo, error)         // get meta information about storage
 	Save(id string, img []byte) error // store image with passed id to staging
 	Load(id string) ([]byte, error)   // load image by ID
+	Delete(id string) error           // delete image by ID
 
 	ResetCleanupTimer(id string) error                    // resets cleanup timer for the image, called on comment preview
 	Commit(id string) error                               // move image from staging to permanent
@@ -90,8 +89,8 @@ func NewService(s Store, p ServiceParams) *Service {
 	return &Service{ServiceParams: p, store: s}
 }
 
-// SubmitAndCommit multiple ids immediately
-func (s *Service) SubmitAndCommit(idsFn func() []string) error {
+// Commit multiple ids immediately
+func (s *Service) Commit(idsFn func() []string) error {
 	errs := new(multierror.Error)
 	for _, id := range idsFn() {
 		err := s.store.Commit(id)
@@ -119,7 +118,7 @@ func (s *Service) Submit(idsFn func() []string) {
 				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.EditDuration {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
-				err := s.SubmitAndCommit(req.idsFn)
+				err := s.Commit(req.idsFn)
 				if err != nil {
 					log.Printf("[WARN] image commit error %v", err)
 				}
@@ -143,39 +142,14 @@ func (s *Service) Submit(idsFn func() []string) {
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
 func (s *Service) ExtractPictures(commentHTML string) (ids []string) {
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
-	if err != nil {
-		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
-		return nil
-	}
-	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
-		if im, ok := sl.Attr("src"); ok {
-			if strings.Contains(im, s.ImageAPI) {
-				elems := strings.Split(im, "/")
-				if len(elems) >= 2 {
-					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
-					ids = append(ids, id)
-				}
-			}
-			if strings.Contains(im, s.ProxyAPI) {
-				proxiedURL, err := url.Parse(im)
-				if err != nil {
-					return
-				}
-				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
-				if err != nil {
-					return
-				}
-				imgID, err := CachedImgID(string(imgURL))
-				if err != nil {
-					return
-				}
-				ids = append(ids, imgID)
-			}
-		}
-	})
+	return s.extractImageIDs(commentHTML, true)
+}
 
-	return ids
+// ExtractNonProxiedPictures gets list of non-proxied images from the doc html and convert from urls to ids, i.e. user/pic.png
+// This method is used in image check on post preview and load, as proxied images have lazy loading
+// and wouldn't be present on disk but still valid as they will be loaded the first time someone requests them.
+func (s *Service) ExtractNonProxiedPictures(commentHTML string) (ids []string) {
+	return s.extractImageIDs(commentHTML, false)
 }
 
 // Cleanup runs periodic cleanup with 1.5*ServiceParams.EditDuration. Blocking loop, should be called inside of goroutine by consumer
@@ -239,6 +213,11 @@ func (s *Service) Load(id string) ([]byte, error) {
 	return s.store.Load(id)
 }
 
+// Delete wraps storage Delete function.
+func (s *Service) Delete(id string) error {
+	return s.store.Delete(id)
+}
+
 // Save wraps storage Save function, validating and resizing the image before calling it.
 func (s *Service) Save(userID string, r io.Reader) (id string, err error) {
 	id = path.Join(userID, guid())
@@ -264,6 +243,43 @@ func (s *Service) ImgContentType(img []byte) string {
 	return contentType
 }
 
+// returns list of image IDs from the comment html, including proxied images if includeProxied is true
+func (s *Service) extractImageIDs(commentHTML string, includeProxied bool) (ids []string) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
+		return nil
+	}
+	doc.Find("img").Each(func(_ int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					ids = append(ids, id)
+				}
+			}
+			if includeProxied && strings.Contains(im, s.ProxyAPI) {
+				proxiedURL, err := url.Parse(im)
+				if err != nil {
+					return
+				}
+				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
+				if err != nil {
+					return
+				}
+				imgID, err := CachedImgID(string(imgURL))
+				if err != nil {
+					return
+				}
+				ids = append(ids, imgID)
+			}
+		}
+	})
+
+	return ids
+}
+
 // prepareImage calls readAndValidateImage and resize on provided image.
 func (s *Service) prepareImage(r io.Reader) ([]byte, error) {
 	data, err := readAndValidateImage(r, s.MaxSize)
@@ -367,7 +383,7 @@ func Sha1Str(s string) string {
 // ID would look like: "cached_images/<sha1-of-image-url-hostname>-<sha1-of-image-entire-url>"
 // <sha1-of-image-url-hostname> - would allow us to identify all images from particular site if ever needed
 // <sha1-of-image-entire-url> - would allow us to avoid storing duplicates of the same image
-//                              (as accurate as deduplication based on potentially mutable url can be)
+// (as accurate as deduplication based on potentially mutable url can be)
 func CachedImgID(imgURL string) (string, error) {
 	parsedURL, err := url.Parse(imgURL)
 	if err != nil {

backend/app/store/image/image_mock.go

@@ -1,115 +1,345 @@
-// Code generated by mockery v1.1.2. DO NOT EDIT.
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package image
 
 import (
-	context "context"
-	time "time"
-
-	mock "github.com/stretchr/testify/mock"
+	"context"
+	"sync"
+	"time"
 )
 
-// MockStore is an autogenerated mock type for the Store type
-type MockStore struct {
-	mock.Mock
-}
+// Ensure, that StoreMock does implement Store.
+// If this is not the case, regenerate this file with moq.
+var _ Store = &StoreMock{}
 
-// Cleanup provides a mock function with given fields: ctx, ttl
-func (_m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
-	ret := _m.Called(ctx, ttl)
+// StoreMock is a mock implementation of Store.
+//
+//	func TestSomethingThatUsesStore(t *testing.T) {
+//
+//		// make and configure a mocked Store
+//		mockedStore := &StoreMock{
+//			CleanupFunc: func(ctx context.Context, ttl time.Duration) error {
+//				panic("mock out the Cleanup method")
+//			},
+//			CommitFunc: func(id string) error {
+//				panic("mock out the Commit method")
+//			},
+//			DeleteFunc: func(id string) error {
+//				panic("mock out the Delete method")
+//			},
+//			InfoFunc: func() (StoreInfo, error) {
+//				panic("mock out the Info method")
+//			},
+//			LoadFunc: func(id string) ([]byte, error) {
+//				panic("mock out the Load method")
+//			},
+//			ResetCleanupTimerFunc: func(id string) error {
+//				panic("mock out the ResetCleanupTimer method")
+//			},
+//			SaveFunc: func(id string, img []byte) error {
+//				panic("mock out the Save method")
+//			},
+//		}
+//
+//		// use mockedStore in code that requires Store
+//		// and then make assertions.
+//
+//	}
+type StoreMock struct {
+	// CleanupFunc mocks the Cleanup method.
+	CleanupFunc func(ctx context.Context, ttl time.Duration) error
 
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, time.Duration) error); ok {
-		r0 = rf(ctx, ttl)
-	} else {
-		r0 = ret.Error(0)
-	}
+	// CommitFunc mocks the Commit method.
+	CommitFunc func(id string) error
 
-	return r0
-}
+	// DeleteFunc mocks the Delete method.
+	DeleteFunc func(id string) error
 
-// Commit provides a mock function with given fields: id
-func (_m *MockStore) Commit(id string) error {
-	ret := _m.Called(id)
+	// InfoFunc mocks the Info method.
+	InfoFunc func() (StoreInfo, error)
 
-	var r0 error
-	if rf, ok := ret.Get(0).(func(string) error); ok {
-		r0 = rf(id)
-	} else {
-		r0 = ret.Error(0)
-	}
+	// LoadFunc mocks the Load method.
+	LoadFunc func(id string) ([]byte, error)
 
-	return r0
-}
+	// ResetCleanupTimerFunc mocks the ResetCleanupTimer method.
+	ResetCleanupTimerFunc func(id string) error
 
-// Info provides a mock function with given fields:
-func (_m *MockStore) Info() (StoreInfo, error) {
-	ret := _m.Called()
+	// SaveFunc mocks the Save method.
+	SaveFunc func(id string, img []byte) error
 
-	var r0 StoreInfo
-	if rf, ok := ret.Get(0).(func() StoreInfo); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(StoreInfo)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func() error); ok {
-		r1 = rf()
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
-// Load provides a mock function with given fields: id
-func (_m *MockStore) Load(id string) ([]byte, error) {
-	ret := _m.Called(id)
-
-	var r0 []byte
-	if rf, ok := ret.Get(0).(func(string) []byte); ok {
-		r0 = rf(id)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]byte)
+	// calls tracks calls to the methods.
+	calls struct {
+		// Cleanup holds details about calls to the Cleanup method.
+		Cleanup []struct {
+			// Ctx is the ctx argument value.
+			Ctx context.Context
+			// TTL is the ttl argument value.
+			TTL time.Duration
+		}
+		// Commit holds details about calls to the Commit method.
+		Commit []struct {
+			// ID is the id argument value.
+			ID string
+		}
+		// Delete holds details about calls to the Delete method.
+		Delete []struct {
+			// ID is the id argument value.
+			ID string
+		}
+		// Info holds details about calls to the Info method.
+		Info []struct {
+		}
+		// Load holds details about calls to the Load method.
+		Load []struct {
+			// ID is the id argument value.
+			ID string
+		}
+		// ResetCleanupTimer holds details about calls to the ResetCleanupTimer method.
+		ResetCleanupTimer []struct {
+			// ID is the id argument value.
+			ID string
+		}
+		// Save holds details about calls to the Save method.
+		Save []struct {
+			// ID is the id argument value.
+			ID string
+			// Img is the img argument value.
+			Img []byte
 		}
 	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string) error); ok {
-		r1 = rf(id)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	lockCleanup           sync.RWMutex
+	lockCommit            sync.RWMutex
+	lockDelete            sync.RWMutex
+	lockInfo              sync.RWMutex
+	lockLoad              sync.RWMutex
+	lockResetCleanupTimer sync.RWMutex
+	lockSave              sync.RWMutex
 }
 
-// ResetCleanupTimer provides a mock function with given fields: id
-func (_m *MockStore) ResetCleanupTimer(id string) error {
-	ret := _m.Called(id)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(string) error); ok {
-		r0 = rf(id)
-	} else {
-		r0 = ret.Error(0)
+// Cleanup calls CleanupFunc.
+func (mock *StoreMock) Cleanup(ctx context.Context, ttl time.Duration) error {
+	if mock.CleanupFunc == nil {
+		panic("StoreMock.CleanupFunc: method is nil but Store.Cleanup was just called")
 	}
-
-	return r0
+	callInfo := struct {
+		Ctx context.Context
+		TTL time.Duration
+	}{
+		Ctx: ctx,
+		TTL: ttl,
+	}
+	mock.lockCleanup.Lock()
+	mock.calls.Cleanup = append(mock.calls.Cleanup, callInfo)
+	mock.lockCleanup.Unlock()
+	return mock.CleanupFunc(ctx, ttl)
 }
 
-// Save provides a mock function with given fields: id, img
-func (_m *MockStore) Save(id string, img []byte) error {
-	ret := _m.Called(id, img)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(string, []byte) error); ok {
-		r0 = rf(id, img)
-	} else {
-		r0 = ret.Error(0)
+// CleanupCalls gets all the calls that were made to Cleanup.
+// Check the length with:
+//
+//	len(mockedStore.CleanupCalls())
+func (mock *StoreMock) CleanupCalls() []struct {
+	Ctx context.Context
+	TTL time.Duration
+} {
+	var calls []struct {
+		Ctx context.Context
+		TTL time.Duration
 	}
+	mock.lockCleanup.RLock()
+	calls = mock.calls.Cleanup
+	mock.lockCleanup.RUnlock()
+	return calls
+}
 
-	return r0
+// Commit calls CommitFunc.
+func (mock *StoreMock) Commit(id string) error {
+	if mock.CommitFunc == nil {
+		panic("StoreMock.CommitFunc: method is nil but Store.Commit was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockCommit.Lock()
+	mock.calls.Commit = append(mock.calls.Commit, callInfo)
+	mock.lockCommit.Unlock()
+	return mock.CommitFunc(id)
+}
+
+// CommitCalls gets all the calls that were made to Commit.
+// Check the length with:
+//
+//	len(mockedStore.CommitCalls())
+func (mock *StoreMock) CommitCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockCommit.RLock()
+	calls = mock.calls.Commit
+	mock.lockCommit.RUnlock()
+	return calls
+}
+
+// Delete calls DeleteFunc.
+func (mock *StoreMock) Delete(id string) error {
+	if mock.DeleteFunc == nil {
+		panic("StoreMock.DeleteFunc: method is nil but Store.Delete was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockDelete.Lock()
+	mock.calls.Delete = append(mock.calls.Delete, callInfo)
+	mock.lockDelete.Unlock()
+	return mock.DeleteFunc(id)
+}
+
+// DeleteCalls gets all the calls that were made to Delete.
+// Check the length with:
+//
+//	len(mockedStore.DeleteCalls())
+func (mock *StoreMock) DeleteCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockDelete.RLock()
+	calls = mock.calls.Delete
+	mock.lockDelete.RUnlock()
+	return calls
+}
+
+// Info calls InfoFunc.
+func (mock *StoreMock) Info() (StoreInfo, error) {
+	if mock.InfoFunc == nil {
+		panic("StoreMock.InfoFunc: method is nil but Store.Info was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockInfo.Lock()
+	mock.calls.Info = append(mock.calls.Info, callInfo)
+	mock.lockInfo.Unlock()
+	return mock.InfoFunc()
+}
+
+// InfoCalls gets all the calls that were made to Info.
+// Check the length with:
+//
+//	len(mockedStore.InfoCalls())
+func (mock *StoreMock) InfoCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockInfo.RLock()
+	calls = mock.calls.Info
+	mock.lockInfo.RUnlock()
+	return calls
+}
+
+// Load calls LoadFunc.
+func (mock *StoreMock) Load(id string) ([]byte, error) {
+	if mock.LoadFunc == nil {
+		panic("StoreMock.LoadFunc: method is nil but Store.Load was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockLoad.Lock()
+	mock.calls.Load = append(mock.calls.Load, callInfo)
+	mock.lockLoad.Unlock()
+	return mock.LoadFunc(id)
+}
+
+// LoadCalls gets all the calls that were made to Load.
+// Check the length with:
+//
+//	len(mockedStore.LoadCalls())
+func (mock *StoreMock) LoadCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockLoad.RLock()
+	calls = mock.calls.Load
+	mock.lockLoad.RUnlock()
+	return calls
+}
+
+// ResetCleanupTimer calls ResetCleanupTimerFunc.
+func (mock *StoreMock) ResetCleanupTimer(id string) error {
+	if mock.ResetCleanupTimerFunc == nil {
+		panic("StoreMock.ResetCleanupTimerFunc: method is nil but Store.ResetCleanupTimer was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockResetCleanupTimer.Lock()
+	mock.calls.ResetCleanupTimer = append(mock.calls.ResetCleanupTimer, callInfo)
+	mock.lockResetCleanupTimer.Unlock()
+	return mock.ResetCleanupTimerFunc(id)
+}
+
+// ResetCleanupTimerCalls gets all the calls that were made to ResetCleanupTimer.
+// Check the length with:
+//
+//	len(mockedStore.ResetCleanupTimerCalls())
+func (mock *StoreMock) ResetCleanupTimerCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockResetCleanupTimer.RLock()
+	calls = mock.calls.ResetCleanupTimer
+	mock.lockResetCleanupTimer.RUnlock()
+	return calls
+}
+
+// Save calls SaveFunc.
+func (mock *StoreMock) Save(id string, img []byte) error {
+	if mock.SaveFunc == nil {
+		panic("StoreMock.SaveFunc: method is nil but Store.Save was just called")
+	}
+	callInfo := struct {
+		ID  string
+		Img []byte
+	}{
+		ID:  id,
+		Img: img,
+	}
+	mock.lockSave.Lock()
+	mock.calls.Save = append(mock.calls.Save, callInfo)
+	mock.lockSave.Unlock()
+	return mock.SaveFunc(id, img)
+}
+
+// SaveCalls gets all the calls that were made to Save.
+// Check the length with:
+//
+//	len(mockedStore.SaveCalls())
+func (mock *StoreMock) SaveCalls() []struct {
+	ID  string
+	Img []byte
+} {
+	var calls []struct {
+		ID  string
+		Img []byte
+	}
+	mock.lockSave.RLock()
+	calls = mock.calls.Save
+	mock.lockSave.RUnlock()
+	return calls
 }

backend/app/store/image/image_test.go

@@ -14,22 +14,30 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
 func TestService_SaveAndLoad(t *testing.T) {
-	store := MockStore{}
+	store := StoreMock{
+		SaveFunc: func(string, []byte) error {
+			return nil
+		},
+		LoadFunc: func(string) ([]byte, error) {
+			return nil, nil
+		},
+	}
 	svc := NewService(&store, ServiceParams{MaxSize: 1500, MaxWidth: 32, MaxHeight: 32})
 
-	store.On("Save", "test_id", mock.Anything).Return(nil)
 	err := svc.SaveWithID("test_id", gopherPNG())
 	assert.NoError(t, err)
+	assert.Equal(t, 1, len(store.SaveCalls()))
+	assert.Equal(t, "test_id", store.SaveCalls()[0].ID)
 
-	store.On("Load", "test_id", mock.Anything).Return(nil, nil)
 	img, err := svc.Load("test_id")
 	assert.NoError(t, err)
 	assert.Nil(t, img)
+	assert.Equal(t, 1, len(store.LoadCalls()))
+	assert.Equal(t, "test_id", store.LoadCalls()[0].ID)
 }
 
 func TestService_Resize(t *testing.T) {
@@ -97,6 +105,7 @@ func TestService_ExtractPictures(t *testing.T) {
 	ids = svc.ExtractPictures(html)
 	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
+	require.Empty(t, svc.ExtractNonProxiedPictures(html), "no non-proxied images expected to be found")
 
 	// bad url
 	html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
@@ -116,72 +125,81 @@ func TestService_ExtractPictures(t *testing.T) {
 }
 
 func TestService_Cleanup(t *testing.T) {
-	store := MockStore{}
-	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
+	store := StoreMock{
+		CleanupFunc: func(context.Context, time.Duration) error {
+			return nil
+		},
+	}
 
 	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
 	// cancel context after 2.1 cleanup TTLs
 	ctx, cancel := context.WithTimeout(context.Background(), svc.EditDuration/100*15*21)
 	defer cancel()
 	svc.Cleanup(ctx)
-	store.AssertNumberOfCalls(t, "Cleanup", 2)
+	assert.Equal(t, 2, len(store.CleanupCalls()))
 }
 
 func TestService_Submit(t *testing.T) {
-	store := MockStore{}
-	store.On("Commit", mock.Anything, mock.Anything).Times(7).Return(nil)
-	store.On("ResetCleanupTimer", mock.Anything, mock.Anything).Times(7).Return(nil)
+	store := StoreMock{
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
+	}
 	svc := NewService(&store, ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100})
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
-	store.AssertNumberOfCalls(t, "ResetCleanupTimer", 3)
-	err := svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} })
+	assert.Equal(t, 3, len(store.ResetCleanupTimerCalls()))
+	err := svc.Commit(func() []string { return []string{"id4", "id5"} })
 	assert.NoError(t, err)
 	svc.Submit(func() []string { return []string{"id6", "id7"} })
-	store.AssertNumberOfCalls(t, "ResetCleanupTimer", 5)
+	assert.Equal(t, 5, len(store.ResetCleanupTimerCalls()))
 	svc.Submit(nil)
-	store.AssertNumberOfCalls(t, "Commit", 2)
+	assert.Equal(t, 2, len(store.CommitCalls()))
 	time.Sleep(time.Millisecond * 175)
-	store.AssertNumberOfCalls(t, "Commit", 7)
+	assert.Equal(t, 7, len(store.CommitCalls()))
 	svc.Close(context.TODO())
 }
 
 func TestService_Close(t *testing.T) {
-	store := MockStore{}
-	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
-	store.On("ResetCleanupTimer", mock.Anything, mock.Anything).Times(5).Return(nil)
+	store := StoreMock{
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
+	}
 	svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
-	store.AssertNumberOfCalls(t, "ResetCleanupTimer", 5)
+	assert.Equal(t, 5, len(store.ResetCleanupTimerCalls()))
 	svc.Close(context.TODO())
-	store.AssertNumberOfCalls(t, "Commit", 5)
+	assert.Equal(t, 5, len(store.CommitCalls()))
 }
 
 func TestService_SubmitDelay(t *testing.T) {
-	store := MockStore{}
-	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
-	store.On("ResetCleanupTimer", mock.Anything, mock.Anything).Times(5).Return(nil)
+	store := StoreMock{
+		CommitFunc: func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error {
+			return nil
+		},
+	}
 	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
 	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
-	store.AssertNumberOfCalls(t, "ResetCleanupTimer", 5)
-	store.AssertNumberOfCalls(t, "Commit", 3)
+	assert.Equal(t, 5, len(store.ResetCleanupTimerCalls()))
+	assert.Equal(t, 3, len(store.CommitCalls()))
 	svc.Close(context.TODO())
-	store.AssertNumberOfCalls(t, "Commit", 5)
+	assert.Equal(t, 5, len(store.CommitCalls()))
 }
 
 func TestService_Info(t *testing.T) {
-	store := MockStore{}
-	store.On("Info", mock.Anything, mock.Anything).Once().Return(StoreInfo{}, nil)
+	store := StoreMock{InfoFunc: func() (StoreInfo, error) {
+		return StoreInfo{}, nil
+	}}
 
 	svc := Service{store: &store, ServiceParams: ServiceParams{}}
 	info, err := svc.Info()
 	assert.NoError(t, err)
 	assert.True(t, info.FirstStagingImageTS.IsZero())
-	store.AssertNumberOfCalls(t, "Info", 1)
+	assert.Equal(t, 1, len(store.InfoCalls()))
 }
 
 func TestService_resize(t *testing.T) {
@@ -259,3 +277,11 @@ func TestCachedImgID(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "cached_images/"+Sha1Str("example.org")+"-"+Sha1Str(imgURL), img)
 }
+
+func TestService_DoubleClose(*testing.T) {
+	store := StoreMock{}
+	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
+	svc.Close(context.TODO())
+	// second call should not result in panic
+	svc.Close(context.TODO())
+}

backend/app/store/image/remote_store.go

@@ -41,6 +41,12 @@ func (r *RPC) Load(id string) ([]byte, error) {
 	return io.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg)))
 }
 
+// Delete image from storage
+func (r *RPC) Delete(id string) error {
+	_, err := r.Call("image.delete", id)
+	return err
+}
+
 // Commit file stored in staging location by moving it to permanent location
 func (r *RPC) Commit(id string) error {
 	_, err := r.Call("image.commit", id)