fix: missing comma in example frontend configuration

This clarifies that the parameter sets CSP 'frame-ancestors'
to limit hosts allowed to embed comments. The commit also improves
the documentation on how to use ALLOWED_HOSTS with AUTH_SAME_SITE
for different setup scenarios.

We might want to change AUTH_SAME_SITE to `strong` in v2.0 as it works
on the subdomain of the same site as well as current Lax option.

.dockerignore

@@ -21,8 +21,10 @@ compose-dev-backend.yml
 compose-dev-frontend.yml
 compose-private-backend.yml
 compose-private-frontend.yml
+compose-e2e-test.yml
 compose-private.yml
 rest-client.env.json
+Makefile
 
 # generated files
 *.cov

.editorconfig

@@ -1,3 +1,5 @@
+root = true
+
 [*]
 indent_style = tab
 insert_final_newline = true

.github/workflows/ci-backend.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: debug if needed
         run: if [[ "$DEBUG" == "true" ]]; then env; fi
@@ -30,36 +30,44 @@ jobs:
           DEBUG: ${{secrets.DEBUG}}
 
       - name: install go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: "1.21"
 
-      - name: install golangci-lint and goveralls
-        run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GITHUB_WORKSPACE v1.49.0
-          go install github.com/mattn/goveralls@latest
-
-      - name: test and lint backend
+      - name: test and build backend
         run: |
           go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
           cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/app
         env:
           TZ: "America/Chicago"
 
-      - name: test and lint examples
+      - name: test examples
         run: |
-          go version
-          $GITHUB_WORKSPACE/golangci-lint version
           go test -race ./...
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/_example/memory_store
         env:
           TZ: "America/Chicago"
 
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          working-directory: backend/app
+
+      - name: golangci-lint on example directory
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          args: --config ../../.golangci.yml
+          working-directory: backend/_example/memory_store
+
       - name: submit coverage
-        run: goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        run: |
+          go install github.com/mattn/goveralls@latest
+          goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
         working-directory: backend
         env:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-build.yml

@@ -29,14 +29,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}

.github/workflows/ci-frontend-api.yml

@@ -20,19 +20,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -64,19 +64,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -108,19 +108,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7

.github/workflows/ci-frontend.yml

@@ -20,19 +20,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -64,19 +64,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -108,19 +108,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -155,39 +155,40 @@ jobs:
       CI_JOB_NUMBER: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
           run_install: false
 
       - name: Check bundle size
-        uses: andresz1/size-limit-action@v1
+        uses: andresz1/size-limit-action@7313b26c76b3666c1dc41e2ca05370e201a9b7de
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           directory: ./frontend/apps/remark42
+          package_manager: pnpm
 
   test:
     name: Tests & Coverage
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7

.github/workflows/ci-site.yml

@@ -20,14 +20,14 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}

.github/workflows/e2e-tests.yml

@@ -6,12 +6,14 @@ on:
     paths:
       - "frontend/apps/remark42/**"
       - "frontend/e2e/**"
+      - "frontend/Dockerfile.e2e"
 
   pull_request:
     branches: [master]
     paths:
       - "frontend/apps/remark42/**"
       - "frontend/e2e/**"
+      - "frontend/Dockerfile.e2e"
 
 jobs:
   tests:
@@ -20,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build & run containers
         id: tests

.gitignore

@@ -25,3 +25,4 @@ compose-private.yml
 /backend/_example/*/vendor
 http-client.env.json
 /playwright-report/
+/backend/app/cmd/var

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM node:16.15.1-alpine AS frontend-deps
+FROM --platform=$BUILDPLATFORM node:16.20-alpine AS frontend-deps
 
 ARG SKIP_FRONTEND_TEST
 ARG SKIP_FRONTEND_BUILD
@@ -11,7 +11,7 @@ COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/
 RUN \
   if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
     apk add --no-cache --update git && \
-    npm i -g pnpm; \
+    npm i -g pnpm@7; \
   fi
 
 RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 \
@@ -45,7 +45,7 @@ RUN \
     echo 'Skip frontend build'; \
   fi
 
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG CI
 ARG GITHUB_REF
@@ -54,15 +54,15 @@ ARG GIT_BRANCH
 ARG SKIP_BACKEND_TEST
 ARG BACKEND_TEST_TIMEOUT
 
+# install gcc in order to be able to go test package with -race
+RUN apk --no-cache add gcc libc-dev
+
 ADD backend /build/backend
 # to embed the frontend files statically into Remark42 binary
 COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
 RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
 WORKDIR /build/backend
 
-# install gcc in order to be able to go test package with -race
-RUN apk --no-cache add gcc libc-dev
-
 RUN echo go version: `go version`
 
 # run tests
@@ -70,7 +70,7 @@ RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
         CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
-        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov && \
         golangci-lint run --config ../.golangci.yml ./... ; \
     else \
       echo "skip backend tests and linter" \
@@ -81,7 +81,7 @@ RUN \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM umputun/baseimage:app-v1.9.2
+FROM umputun/baseimage:app-v1.13.0
 
 ARG GITHUB_SHA
 
@@ -89,7 +89,7 @@ LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
       org.opencontainers.image.description="Remark42 comment engine" \
       org.opencontainers.image.documentation="https://remark42.com/docs/getting-started/" \
       org.opencontainers.image.licenses="MIT" \
-      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
       org.opencontainers.image.title="Remark42" \
       org.opencontainers.image.url="https://remark42.com/" \
       org.opencontainers.image.revision="${GITHUB_SHA}"

Dockerfile.artifacts

@@ -1,4 +1,4 @@
-FROM node:16.15.1-alpine AS frontend-deps
+FROM node:16.20.2-alpine AS frontend-deps
 
 ENV CI=true
 
@@ -7,7 +7,7 @@ WORKDIR /srv/frontend
 COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
 COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/package.json
 
-RUN apk add --no-cache --update git && npm i -g pnpm
+RUN apk add --no-cache --update git && npm i -g pnpm@7
 RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 pnpm i
 
 FROM frontend-deps AS build-frontend
@@ -19,7 +19,7 @@ WORKDIR /srv/frontend/apps/remark42/
 COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
 RUN pnpm build
 
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG GITHUB_TOKEN
 ARG GITHUB_REF

Makefile

@@ -39,14 +39,17 @@ race_test:
 	cd backend/app && go test -race -timeout=60s -count 1 ./...
 
 backend:
-	docker-compose -f compose-dev-backend.yml build
+	docker compose -f compose-dev-backend.yml build
 
 frontend:
-	docker-compose -f compose-dev-frontend.yml build
+	docker compose -f compose-dev-frontend.yml build
 
 rundev:
 	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true GITHUB_REF=$(GITHUB_REF) GITHUB_SHA=$(GITHUB_SHA) CI=true \
-		docker-compose -f compose-private.yml build
-	docker-compose -f compose-private.yml up
+		docker compose -f compose-private.yml build
+	docker compose -f compose-private.yml up
+
+e2e:
+	docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
 
 .PHONY: bin backend

README.md

@@ -2,7 +2,7 @@
 
 Remark42 is a self-hosted, lightweight and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles, or any other place where readers add comments.
 
-* Social login via Google, Twitter, Facebook, Microsoft, GitHub, Yandex, Patreon and Telegram
+* Social login via Google, Facebook, Microsoft, GitHub, Apple, Yandex, Patreon and Telegram
 * Login via email
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations

backend/.golangci.yml

@@ -1,15 +1,10 @@
 run:
   timeout: 5m
-  output:
-    format: tab
-  skip-dirs:
-    - vendor
 
 linters-settings:
   govet:
-    check-shadowing: true
-  maligned:
-    suggest-new: true
+    enable:
+      - shadow
   goconst:
     min-len: 2
     min-occurrences: 2
@@ -31,29 +26,31 @@ linters-settings:
 linters:
   enable:
     - bodyclose
-    - megacheck
-    - revive
-    - govet
-    - unconvert
-    - gas
-    - gocyclo
     - dupl
+    - exportloopref
+    - gochecknoinits
+    - gocritic
+    - gocyclo
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
     - misspell
+    - nakedret
+    - prealloc
+    - revive
+    - staticcheck
+    - stylecheck
+    - typecheck
+    - unconvert
     - unparam
     - unused
-    - typecheck
-    - ineffassign
-    - stylecheck
-    - gochecknoinits
-    - exportloopref
-    - gocritic
-    - nakedret
-    - gosimple
-    - prealloc
   fast: false
   disable-all: true
 
 issues:
+  exclude-dirs:
+    - vendor
   exclude-rules:
     - text: "at least one file in a package should have a package comment"
       linters:

backend/_example/memory_store/Dockerfile

@@ -1,11 +1,11 @@
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ADD backend /build/backend
 WORKDIR /build/backend/_example/memory_store
 
 RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
 
-FROM umputun/baseimage:app-v1.9.2
+FROM umputun/baseimage:app-v1.13.0
 
 ARG GITHUB_SHA
 
@@ -13,7 +13,7 @@ LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
       org.opencontainers.image.description="Remark42 comment engine example JRPC memory store" \
       org.opencontainers.image.documentation="https://github.com/umputun/remark42/tree/master/backend/_example/memory_store" \
       org.opencontainers.image.licenses="MIT" \
-      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
       org.opencontainers.image.title="Remark42 JRPC example memory store" \
       org.opencontainers.image.url="https://remark42.com/" \
       org.opencontainers.image.revision="${GITHUB_SHA}"

backend/_example/memory_store/README.md

@@ -4,9 +4,9 @@
 
 In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
 
-1. `docker-compose -f compose-dev-memstore.yml build`
-1. `docker-compose -f compose-dev-memstore.yml up`
+1. `docker compose -f compose-dev-memstore.yml build`
+1. `docker compose -f compose-dev-memstore.yml up`
 
 As usual, demo site will run on http://127.0.0.1:8080/web/
 
-note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -35,7 +35,8 @@ func NewMemAdminStore(key string) *MemAdmin {
 	return &MemAdmin{data: map[string]AdminRec{}, key: key}
 }
 
-// Key executes find by siteID and returns substructure with secret key
+// Key supposed to execute find by siteID and returns substructure with secret key,
+// but in this case the shared secret is used for all sites
 func (m *MemAdmin) Key(_ string) (key string, err error) {
 	return m.key, nil
 }

backend/_example/memory_store/accessor/data.go

@@ -293,17 +293,17 @@ func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailE
 		defer m.mu.Unlock()
 
 		if req.Update == "" { // read detail value, no update requested
-			return m.getUserDetail(req)
+			return m.getUserDetail(req), nil
 		}
 
-		return m.setUserDetail(req)
+		return m.setUserDetail(req), nil
 	case engine.AllUserDetails:
 		// list of all details returned in case request is a read request
 		// (Update is not set) and does not have UserID or Detail set
 		if req.Update == "" && req.UserID == "" { // read list of all details
 			m.mu.Lock()
 			defer m.mu.Unlock()
-			return m.listDetails(req.Locator)
+			return m.listDetails(req.Locator), nil
 		}
 		return nil, fmt.Errorf("unsupported request with userdetail all")
 	default:
@@ -319,7 +319,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 
 	switch {
 	case req.UserDetail != "": // delete user detail
-		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		return nil
 	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
 		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
 
@@ -332,7 +333,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 				return e
 			}
 		}
-		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		return nil
 
 	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
 		if _, ok := m.posts[req.Locator.SiteID]; !ok {
@@ -437,29 +439,29 @@ func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
 
 // getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
 // as an only element of the slice.
-func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		switch req.Detail {
 		case engine.UserEmail:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}
 		case engine.UserTelegram:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}
 		}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
 // element of the slice in case of success
-func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	var entry metaUser
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		entry = meta
 	}
@@ -476,42 +478,42 @@ func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDeta
 	case engine.UserEmail:
 		entry.Details.Email = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}
 	case engine.UserTelegram:
 		entry.Details.Telegram = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // listDetails lists all available users details for given siteID
-func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+func (m *MemData) listDetails(loc store.Locator) []engine.UserDetailEntry {
 	var res []engine.UserDetailEntry
 	for _, u := range m.metaUsers {
 		if u.SiteID == loc.SiteID {
 			res = append(res, u.Details)
 		}
 	}
-	return res, nil
+	return res
 }
 
 // deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
 // deletion of the absent entry doesn't produce error.
 // Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
-func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) {
 	var entry metaUser
 	if meta, ok := m.metaUsers[userID]; ok {
 		if meta.SiteID != locator.SiteID {
-			return nil
+			return
 		}
 		entry = meta
 	}
 
 	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
 		// absent entry means that we should not do anything
-		return nil
+		return
 	}
 
 	switch userDetail {
@@ -529,7 +531,6 @@ func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDet
 	}
 
 	m.metaUsers[userID] = entry
-	return nil
 }
 
 func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {

backend/_example/memory_store/accessor/data_test.go

@@ -547,12 +547,12 @@ func TestMemData_FlagListBlocked(t *testing.T) {
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	var blockedIds = make([]string, len(blockedList))
+	var blockedIDs = make([]string, len(blockedList))
 	for i, x := range blockedList {
-		blockedIds[i] = x.ID
+		blockedIDs[i] = x.ID
 	}
 	require.Equal(t, 2, len(blockedList), b.metaUsers)
-	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIDs)
 	t.Logf("%+v", blockedList)
 
 	// check block expiration
@@ -624,6 +624,7 @@ func TestMemData_DeleteComment(t *testing.T) {
 func TestMemData_Close(t *testing.T) {
 	b := prepMem(t)
 	assert.NoError(t, b.Close())
+	assert.NoError(t, b.Close(), "second call should not result in panic or errors")
 }
 
 func TestMemData_DeleteHard(t *testing.T) {

backend/_example/memory_store/accessor/image.go

@@ -70,6 +70,17 @@ func (m *MemImage) Load(id string) ([]byte, error) {
 	return img, nil
 }
 
+// Delete image by ID
+func (m *MemImage) Delete(id string) error {
+	m.mu.Lock()
+	// delete key from permanent and staging storage
+	delete(m.images, id)
+	delete(m.insertTime, id)
+	delete(m.imagesStaging, id)
+	m.mu.Unlock()
+	return nil
+}
+
 // Commit moves image from staging to permanent
 func (m *MemImage) Commit(id string) error {
 	m.mu.RLock()

backend/_example/memory_store/accessor/image_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 // gopher png for test, from https://golang.org/src/image/png/example_test.go
-const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+const rawGopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
 	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
 	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
 	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
@@ -38,7 +38,9 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
 	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
 	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
-func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNG() io.Reader {
+	return base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawGopher))
+}
 
 func TestMemImage_LoadAfterSave(t *testing.T) {
 	svc := NewMemImageStore()
@@ -57,7 +59,8 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, gopher, img)
 
-	svc.ResetCleanupTimer(id)
+	err = svc.ResetCleanupTimer(id)
+	assert.NoError(t, err)
 
 	err = svc.Commit(id)
 	assert.NoError(t, err)
@@ -70,6 +73,26 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.Equal(t, gopher, img)
 }
 
+func TestMemImage_LoadAfterDelete(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	img, err := svc.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+	assert.Empty(t, img)
+
+	err = svc.ResetCleanupTimer(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
 func TestMemImage_CommitFail(t *testing.T) {
 	svc := NewMemImageStore()
 	err := svc.Commit("test_id")

backend/_example/memory_store/go.mod

@@ -1,42 +1,43 @@
 module github.com/umputun/remark42/memory_store
 
-go 1.17
+go 1.21
 
 require (
 	github.com/go-pkgz/jrpc v0.3.0
-	github.com/go-pkgz/lgr v0.10.4
-	github.com/jessevdk/go-flags v1.5.0
-	github.com/stretchr/testify v1.8.0
-	github.com/umputun/remark42/backend v1.10.1
+	github.com/go-pkgz/lgr v0.11.1
+	github.com/jessevdk/go-flags v1.6.1
+	github.com/stretchr/testify v1.9.0
+	github.com/umputun/remark42/backend v1.1000.0
 )
 
 require (
 	github.com/Depado/bfchroma/v2 v2.0.0 // indirect
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/PuerkitoBio/goquery v1.9.2 // indirect
 	github.com/ajg/form v1.5.1 // indirect
-	github.com/alecthomas/chroma/v2 v2.3.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/alecthomas/chroma/v2 v2.13.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/didip/tollbooth/v7 v7.0.1 // indirect
+	github.com/didip/tollbooth/v7 v7.0.2 // indirect
 	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.7 // indirect
-	github.com/go-chi/render v1.0.2 // indirect
-	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
-	github.com/go-pkgz/rest v1.16.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/go-chi/chi/v5 v5.1.0 // indirect
+	github.com/go-chi/render v1.0.3 // indirect
+	github.com/go-pkgz/expirable-cache/v3 v3.0.0 // indirect
+	github.com/go-pkgz/rest v1.19.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.20 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rs/xid v1.4.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/image v0.0.0-20220617043117-41969df76e82 // indirect
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	go.etcd.io/bbolt v1.3.10 // indirect
+	golang.org/x/image v0.18.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/umputun/remark42/backend => ../../
+replace github.com/umputun/remark42/backend v1.1000.0 => ../../

backend/_example/memory_store/go.sum

@@ -1,90 +1,127 @@
 github.com/Depado/bfchroma/v2 v2.0.0 h1:IRpN9BPkNwEpR6w1ectIcNWOuhDSLx+8f1pn83fzxx8=
 github.com/Depado/bfchroma/v2 v2.0.0/go.mod h1:wFwW/Pw8Tnd0irzgO9Zxtxgzp3aPS8qBWlyadxujxmw=
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
-github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
 github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
-github.com/alecthomas/chroma/v2 v2.3.0 h1:83xfxrnjv8eK+Cf8qZDzNo3PPF9IbTWHs7z28GY6D0U=
-github.com/alecthomas/chroma/v2 v2.3.0/go.mod h1:mZxeWZlxP2Dy+/8cBob2PYd8O2DwNAzave5AY7A2eQw=
-github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
+github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/didip/tollbooth/v7 v7.0.0/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
-github.com/didip/tollbooth/v7 v7.0.1 h1:TkT4sBKoQoHQFPf7blQ54iHrZiTDnr8TceU+MulVAog=
-github.com/didip/tollbooth/v7 v7.0.1/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
+github.com/didip/tollbooth/v7 v7.0.2 h1:WYEfusYI6g64cN0qbZgekDrYfuYBZjUZd5+RlWi69p4=
+github.com/didip/tollbooth/v7 v7.0.2/go.mod h1:RtRYfEmFGX70+ike5kSndSvLtQ3+F2EAmTI4Un/VXNc=
 github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f h1:jtKwihcLmUC9BAhoJ9adCUqdSSZcOdH2KL7mPTUm2aw=
 github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f/go.mod h1:q9C80dnsuVRP2dAskjnXRNWdUJqtGgwG9wNrzt0019s=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
-github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
+github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg=
-github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
-github.com/go-pkgz/expirable-cache v0.1.0 h1:3bw0m8vlTK8qlwz5KXuygNBTkiKRTPrAGXU0Ej2AC1g=
+github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
+github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
 github.com/go-pkgz/expirable-cache v0.1.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
+github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw=
+github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec=
 github.com/go-pkgz/jrpc v0.3.0 h1:Fls38KqPsHzvp0FWfivr6cGnncC+iFBodHBqvUPY+0U=
 github.com/go-pkgz/jrpc v0.3.0/go.mod h1:MFtKs75JESiSqVicsQkgN2iDFFuCd3gVT1/vKiwRi00=
-github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
-github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
+github.com/go-pkgz/lgr v0.11.1 h1:hXFhZcznehI6imLhEa379oMOKFz7TQUmisAqb3oLOSM=
+github.com/go-pkgz/lgr v0.11.1/go.mod h1:tgDF4RXQnBfIgJqjgkv0yOeTQ3F1yewWIZkpUhHnAkU=
 github.com/go-pkgz/rest v1.15.6/go.mod h1:KUWAqbDteYGS/CiXftomQsKjtEOifXsJ36Ka0skYbmk=
-github.com/go-pkgz/rest v1.16.0 h1:3An51XJXUq1XtICHvfAEp048ZCbUHBEga3g0WEu3zK0=
-github.com/go-pkgz/rest v1.16.0/go.mod h1:HHlLOt02NJc2sgffXBF6hYVMcRo4Gz3vjg43zTzN7VM=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/go-pkgz/rest v1.19.0 h1:FNMi5QX5dDIkuC+/e0r+CWsTuOTwUiWMRSA16Ou+9+A=
+github.com/go-pkgz/rest v1.19.0/go.mod h1:Po+W6zQzpMPP6XDGLdAN2aW7UKk1IyrLSb48Lp1N3oQ=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
-github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
+github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
-github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
-golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
+go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/server/data_test.go

@@ -338,6 +338,6 @@ func TestRPC_closeHndl(t *testing.T) {
 	api := fmt.Sprintf("http://localhost:%d/test", port)
 
 	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
-	err := re.Close()
-	assert.NoError(t, err)
+	assert.NoError(t, re.Close())
+	assert.NoError(t, re.Close(), "second call should not result in panic or errors")
 }

backend/_example/memory_store/server/image.go

@@ -35,7 +35,6 @@ func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.R
 	}
 	err := s.img.ResetCleanupTimer(fileID)
 	return jrpc.EncodeResponse(id, nil, err)
-
 }
 
 func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
@@ -47,6 +46,16 @@ func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response)
 	return jrpc.EncodeResponse(id, value, err)
 }
 
+func (s *RPC) imgDeleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Delete(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+
+}
+
 func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
 	var fileID string
 	if err := json.Unmarshal(params, &fileID); err != nil {

backend/_example/memory_store/server/image_test.go

@@ -158,4 +158,9 @@ func TestRPC_imgInfoHndl(t *testing.T) {
 	info, err = ri.Info()
 	assert.NoError(t, err)
 	assert.False(t, info.FirstStagingImageTS.IsZero())
+
+	err = ri.Delete("test_img")
+	assert.NoError(t, err)
+	_, err = ri.Load("test_img")
+	assert.EqualError(t, err, "image test_img not found")
 }

backend/_example/memory_store/server/rpc.go

@@ -60,6 +60,7 @@ func (s *RPC) addHandlers() {
 		"save_with_id":        s.imgSaveWithIDHndl,
 		"reset_cleanup_timer": s.imgResetClnTimerHndl,
 		"load":                s.imgLoadHndl,
+		"delete":              s.imgDeleteHndl,
 		"commit":              s.imgCommitHndl,
 		"cleanup":             s.imgCleanupHndl,
 		"info":                s.imgInfoHndl,

backend/app/cmd/backup_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +18,10 @@ func TestBackup_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
 		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		fmt.Fprint(w, "blah\nblah2\n12345678\n")
 	}))
 	defer ts.Close()
@@ -34,6 +40,28 @@ func TestBackup_Execute(t *testing.T) {
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
+func TestBackup_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		require.Equal(t, "admin:", string(auth))
+		w.WriteHeader(http.StatusUnauthorized)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestBackup_ExecuteFailedStatus(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")

backend/app/cmd/cleanup_test.go

@@ -195,7 +195,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
 	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
@@ -203,7 +203,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		c.lock.Unlock()
 	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()

backend/app/cmd/cmd.go

@@ -115,6 +115,9 @@ func responseError(resp *http.Response) error {
 	if e != nil {
 		body = []byte("")
 	}
+	if resp.StatusCode == http.StatusUnauthorized {
+		return fmt.Errorf("error response %q, ensure you have set ADMIN_PASSWD and provided it to the command you're running: %s", resp.Status, body)
+	}
 	return fmt.Errorf("error response %q, %s", resp.Status, body)
 }
 

backend/app/cmd/import_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -18,6 +20,10 @@ func TestImport_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
@@ -46,6 +52,42 @@ func TestImport_Execute(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestImport_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestImport_ExecuteFailed(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")

backend/app/cmd/remap_test.go

@@ -1,9 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +19,10 @@ func TestRemap_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
@@ -33,3 +40,31 @@ func TestRemap_Execute(t *testing.T) {
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
+
+func TestRemap_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}

backend/app/cmd/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"embed"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,7 +16,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/jrpc"
-	"github.com/go-pkgz/lcw/eventbus"
+	"github.com/go-pkgz/lcw/v2/eventbus"
 	log "github.com/go-pkgz/lgr"
 	ntf "github.com/go-pkgz/notify"
 	"github.com/golang-jwt/jwt"
@@ -27,7 +28,7 @@ import (
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 
 	"github.com/umputun/remark42/backend/app/migrator"
 	"github.com/umputun/remark42/backend/app/notify"
@@ -58,34 +59,36 @@ type ServerCommand struct {
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
-	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
-	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
-	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
-	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	AdminEdit        bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
-	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	Address          string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
-	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
-	RestrictedNames  []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
-	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
-	SimpleView       bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
-	ProxyCORS        bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
-	AllowedHosts     []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
-	SubscribersOnly  bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
-	DisableSignature bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	Sites                      []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AnonymousVote              bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AdminPasswd                string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation             string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles             int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	LegacyImageProxy           bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	MinCommentSize             int           `long:"min-comment" env:"MIN_COMMENT_SIZE" default:"0" description:"min comment size"`
+	MaxCommentSize             int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes                   int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	RestrictVoteIP             bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	DurationVoteIP             time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	LowScore                   int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore              int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore              bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge                int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration               time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	AdminEdit                  bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
+	Port                       int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	Address                    string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
+	WebRoot                    string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit                float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords            []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedNames            []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
+	EnableEmoji                bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView                 bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
+	ProxyCORS                  bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
+	AllowedHosts               []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments via CSP 'frame-ancestors''" env-delim:","`
+	SubscribersOnly            bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
+	DisableSignature           bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	DisableFancyTextFormatting bool          `long:"disable-fancy-text-formatting" env:"DISABLE_FANCY_TEXT_FORMATTING" description:"disable fancy comments text formatting (replacement of quotes, dashes, fractions, etc)"`
 
 	Auth struct {
 		TTL struct {
@@ -96,16 +99,17 @@ type ServerCommand struct {
 		SendJWTHeader bool   `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"send JWT as a header instead of cookie"`
 		SameSite      string `long:"same-site" env:"SAME_SITE" description:"set same site policy for cookies" choice:"default" choice:"none" choice:"lax" choice:"strict" default:"default"` // nolint
 
-		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Microsoft AuthGroup `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
-		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Twitter   AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
-		Patreon   AuthGroup `group:"patreon" namespace:"patreon" env-namespace:"PATREON" description:"Patreon OAuth"`
-		Telegram  bool      `long:"telegram" env:"TELEGRAM" description:"Enable Telegram auth (using token from telegram.token)"`
-		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
-		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Apple     AppleGroup `group:"apple" namespace:"apple" env-namespace:"APPLE" description:"Apple OAuth"`
+		Google    AuthGroup  `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Github    AuthGroup  `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Facebook  AuthGroup  `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Microsoft AuthGroup  `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
+		Yandex    AuthGroup  `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Twitter   AuthGroup  `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"[deprecated, doesn't work] Twitter OAuth"`
+		Patreon   AuthGroup  `group:"patreon" namespace:"patreon" env-namespace:"PATREON" description:"Patreon OAuth"`
+		Telegram  bool       `long:"telegram" env:"TELEGRAM" description:"Enable Telegram auth (using token from telegram.token)"`
+		Dev       bool       `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool       `long:"anon" env:"ANON" description:"enable anonymous login"`
 		Email     struct {
 			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
 			From         string        `long:"from" env:"FROM" description:"from email address"`
@@ -133,6 +137,14 @@ type ImageProxyGroup struct {
 	CacheExternal bool `long:"cache-external" env:"CACHE_EXTERNAL" description:"enable caching for external images"`
 }
 
+// AppleGroup defines options for Apple auth params
+type AppleGroup struct {
+	CID                string `long:"cid" env:"CID" description:"Apple client ID (App ID or Services ID)"`
+	TID                string `long:"tid" env:"TID" description:"Apple service ID"`
+	KID                string `long:"kid" env:"KID" description:"Private key ID"`
+	PrivateKeyFilePath string `long:"private-key-filepath" env:"PRIVATE_KEY_FILEPATH" description:"Private key file location" default:"/srv/var/apple.p8"`
+}
+
 // AuthGroup defines options group for auth params
 type AuthGroup struct {
 	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
@@ -197,7 +209,7 @@ type AdminGroup struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  []string `long:"email" env:"EMAIL" description:"admin emails" env-delim:","`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
-	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
+	RPC AdminRPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // TelegramGroup defines token for Telegram used in notify and auth modules
@@ -208,14 +220,15 @@ type TelegramGroup struct {
 
 // SMTPGroup defines options for SMTP server connection, used in auth and notify modules
 type SMTPGroup struct {
-	Host      string        `long:"host" env:"HOST" description:"SMTP host"`
-	Port      int           `long:"port" env:"PORT" description:"SMTP port"`
-	Username  string        `long:"username" env:"USERNAME" description:"SMTP user name"`
-	Password  string        `long:"password" env:"PASSWORD" description:"SMTP password"`
-	TLS       bool          `long:"tls" env:"TLS" description:"enable TLS"`
-	LoginAuth bool          `long:"login_auth" env:"LOGIN_AUTH" description:"enable LOGIN auth instead of PLAIN"`
-	StartTLS  bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
-	TimeOut   time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
+	Host               string        `long:"host" env:"HOST" description:"SMTP host"`
+	Port               int           `long:"port" env:"PORT" description:"SMTP port"`
+	Username           string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Password           string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	TLS                bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	InsecureSkipVerify bool          `long:"insecure_skip_verify" env:"INSECURE_SKIP_VERIFY" description:"skip certificate verification"`
+	LoginAuth          bool          `long:"login_auth" env:"LOGIN_AUTH" description:"enable LOGIN auth instead of PLAIN"`
+	StartTLS           bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
+	TimeOut            time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
 }
 
 // NotifyGroup defines options for notification
@@ -265,6 +278,12 @@ type RPCGroup struct {
 	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
 }
 
+// AdminRPCGroup defines options for remote admin store
+type AdminRPCGroup struct {
+	RPCGroup
+	SecretPerSite bool `long:"secret_per_site" env:"SECRET_PER_SITE" description:"enable JWT secret retrieval per aud, which is site_id in this case"`
+}
+
 // LoadingCache defines interface for caching
 type LoadingCache interface {
 	Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
@@ -294,6 +313,7 @@ func (s *ServerCommand) Execute(_ []string) error {
 	log.Printf("[INFO] start server on port %s:%d", s.Address, s.Port)
 	resetEnv(
 		"SECRET",
+		"AUTH_APPLE_KID",
 		"AUTH_GOOGLE_CSEC",
 		"AUTH_GITHUB_CSEC",
 		"AUTH_FACEBOOK_CSEC",
@@ -387,6 +407,12 @@ func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
 	if s.Notify.Telegram.API != "https://api.telegram.org/bot" {
 		result = append(result, DeprecatedFlag{Old: "notify.telegram.api", Version: "1.9"})
 	}
+	if s.Auth.Twitter.CID != "" {
+		result = append(result, DeprecatedFlag{Old: "auth.twitter.cid", Version: "1.14"})
+	}
+	if s.Auth.Twitter.CSEC != "" {
+		result = append(result, DeprecatedFlag{Old: "auth.twitter.csec", Version: "1.14"})
+	}
 	return append(result, s.findDeprecatedFlagsCollisions()...)
 }
 
@@ -485,11 +511,12 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		EditDuration:           s.EditDuration,
 		AdminEdits:             s.AdminEdit,
 		AdminStore:             adminStore,
+		MinCommentSize:         s.MinCommentSize,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
 		PositiveScore:          s.PositiveScore,
 		ImageService:           imageService,
-		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}, s.getAllowedDomains()),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
 	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
@@ -510,11 +537,12 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	authenticator := s.getAuthenticator(dataService, avatarStore, adminStore, authRefreshCache)
 
 	telegramAuth := s.makeTelegramAuth(authenticator) // telegram auth requires TelegramAPI listener which is constructed below
-	telegramService, telegramBotUsername := s.startTelegramAuthAndNotify(ctx, telegramAuth)
+	telegramService := s.startTelegramAuthAndNotify(ctx, telegramAuth)
 
 	err = s.addAuthProviders(authenticator)
 	if err != nil {
 		_ = dataService.Close()
+		_ = authRefreshCache.Close()
 		return nil, fmt.Errorf("failed to make authenticator: %w", err)
 	}
 
@@ -524,7 +552,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		Cache:             loadingCache,
 		NativeImporter:    &migrator.Native{DataStore: dataService},
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
-		WordPressImporter: &migrator.WordPress{DataStore: dataService},
+		WordPressImporter: &migrator.WordPress{DataStore: dataService, DisableFancyTextFormatting: s.DisableFancyTextFormatting},
 		CommentoImporter:  &migrator.Commento{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
 		URLMapperMaker:    migrator.NewURLMapper,
@@ -554,37 +582,40 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
 		_ = dataService.Close()
+		_ = authRefreshCache.Close()
 		return nil, fmt.Errorf("failed to make config of ssl server params: %w", err)
 	}
 
 	srv := &api.Rest{
-		Version:             s.Revision,
-		DataService:         dataService,
-		WebRoot:             s.WebRoot,
-		WebFS:               webFS,
-		RemarkURL:           s.RemarkURL,
-		ImageProxy:          imgProxy,
-		CommentFormatter:    commentFormatter,
-		Migrator:            migr,
-		ReadOnlyAge:         s.ReadOnlyAge,
-		SharedSecret:        s.SharedSecret,
-		Authenticator:       authenticator,
-		Cache:               loadingCache,
-		NotifyService:       notifyService,
-		TelegramService:     telegramService,
-		SSLConfig:           sslConfig,
-		UpdateLimiter:       s.UpdateLimit,
-		ImageService:        imageService,
-		EmailNotifications:  contains("email", s.Notify.Users),
-		TelegramBotUsername: telegramBotUsername,
-		EmojiEnabled:        s.EnableEmoji,
-		AnonVote:            s.AnonymousVote && s.RestrictVoteIP,
-		SimpleView:          s.SimpleView,
-		ProxyCORS:           s.ProxyCORS,
-		AllowedAncestors:    s.AllowedHosts,
-		SendJWTHeader:       s.Auth.SendJWTHeader,
-		SubscribersOnly:     s.SubscribersOnly,
-		DisableSignature:    s.DisableSignature,
+		Version:                    s.Revision,
+		DataService:                dataService,
+		WebRoot:                    s.WebRoot,
+		WebFS:                      webFS,
+		RemarkURL:                  s.RemarkURL,
+		ImageProxy:                 imgProxy,
+		CommentFormatter:           commentFormatter,
+		Migrator:                   migr,
+		ReadOnlyAge:                s.ReadOnlyAge,
+		SharedSecret:               s.SharedSecret,
+		Authenticator:              authenticator,
+		Cache:                      loadingCache,
+		NotifyService:              notifyService,
+		TelegramService:            telegramService,
+		SSLConfig:                  sslConfig,
+		UpdateLimiter:              s.UpdateLimit,
+		ImageService:               imageService,
+		EmailNotifications:         contains("email", s.Notify.Users),
+		TelegramNotifications:      contains("telegram", s.Notify.Users) && telegramService != nil,
+		EmojiEnabled:               s.EnableEmoji,
+		AnonVote:                   s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:                 s.SimpleView,
+		ProxyCORS:                  s.ProxyCORS,
+		AllowedAncestors:           s.AllowedHosts,
+		SendJWTHeader:              s.Auth.SendJWTHeader,
+		SubscribersOnly:            s.SubscribersOnly,
+		DisableSignature:           s.DisableSignature,
+		DisableFancyTextFormatting: s.DisableFancyTextFormatting,
+		ExternalImageProxy:         s.ImageProxy.CacheExternal,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -594,6 +625,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		da, errDevAuth := authenticator.DevAuth()
 		if errDevAuth != nil {
 			_ = dataService.Close()
+			_ = authRefreshCache.Close()
 			return nil, fmt.Errorf("can't make dev oauth2 server: %w", errDevAuth)
 		}
 		devAuth = da
@@ -615,6 +647,47 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	}, nil
 }
 
+// Extract domains from s.AllowedHosts and second level domain from s.RemarkURL.
+// It can be and IP like http://127.0.0.1 in which case we need to use whole IP as domain
+// Beware, if s.RemarkURL is in third-level domain like https://example.co.uk, co.uk will be returned.
+func (s *ServerCommand) getAllowedDomains() []string {
+	rawDomains := s.AllowedHosts
+	rawDomains = append(rawDomains, s.RemarkURL)
+	allowedDomains := []string{}
+	for _, rawURL := range rawDomains {
+		// case of 'self' AllowedHosts, which is not a valid rawURL name
+		if rawURL == "self" || rawURL == "'self'" || rawURL == "\"self\"" {
+			continue
+		}
+		// AllowedHosts usually don't have https:// prefix, so we're adding it just to make parsing below work the same way as for RemarkURL
+		if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
+			rawURL = "https://" + rawURL
+		}
+		parsedURL, err := url.Parse(rawURL)
+		if err != nil {
+			log.Printf("[WARN] failed to parse URL %s for TitleExtract whitelist: %v", rawURL, err)
+			continue
+		}
+		domain := parsedURL.Hostname()
+
+		if domain == "" || // don't add empty domain as it will allow everything to be extracted
+			(len(strings.Split(domain, ".")) < 2 && // don't allow single-word domains like "com"
+				domain != "localhost") { // localhost is an exceptional single-word domain which is allowed
+			continue
+		}
+
+		// Only for RemarkURL if domain is not IP and has more than two levels, extract second level domain.
+		// For AllowedHosts we don't do this as they are exact list of domains which can host comments, but
+		// RemarkURL might be on a subdomain and we must allow parent domain to be used for TitleExtract.
+		if rawURL == s.RemarkURL && net.ParseIP(domain) == nil && len(strings.Split(domain, ".")) > 2 {
+			domain = strings.Join(strings.Split(domain, ".")[len(strings.Split(domain, "."))-2:], ".")
+		}
+
+		allowedDomains = append(allowedDomains, domain)
+	}
+	return allowedDomains
+}
+
 // Run all application objects
 func (a *serverApp) run(ctx context.Context) error {
 	if a.AdminPasswd != "" {
@@ -804,37 +877,53 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 
 func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
+	o := cache.NewOpts[[]byte]()
 	switch s.Cache.Type {
 	case "redis_pub_sub":
 		redisPubSub, err := eventbus.NewRedisPubSub(s.Cache.RedisAddr, "remark42-cache")
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization, redis PubSub initialisation: %w", err)
 		}
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items), cache.EventBus(redisPubSub))
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
+			o.MaxKeys(s.Cache.Max.Items), o.EventBus(redisPubSub))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "mem":
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items))
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
+			o.MaxKeys(s.Cache.Max.Items))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "none":
-		return cache.NewScache(&cache.Nop{}), nil
+		return cache.NewScache[[]byte](&cache.Nop[[]byte]{}), nil
 	}
 	return nil, fmt.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
+//nolint:gocyclo // simple code but many if checks
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 	providersCount := 0
 	if s.Auth.Telegram {
 		providersCount++
 	}
 
+	if s.Auth.Apple.CID != "" && s.Auth.Apple.TID != "" && s.Auth.Apple.KID != "" {
+		err := authenticator.AddAppleProvider(
+			provider.AppleConfig{
+				ClientID: s.Auth.Apple.CID,
+				TeamID:   s.Auth.Apple.TID,
+				KeyID:    s.Auth.Apple.KID,
+			},
+			provider.LoadApplePrivateKeyFromFile(s.Auth.Apple.PrivateKeyFilePath),
+		)
+		if err != nil {
+			return err
+		}
+		providersCount++
+	}
 	if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
 		authenticator.AddProvider("google", s.Auth.Google.CID, s.Auth.Google.CSEC)
 		providersCount++
@@ -876,18 +965,19 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 
 	if s.Auth.Email.Enable {
 		params := sender.EmailParams{
-			Host:         s.SMTP.Host,
-			Port:         s.SMTP.Port,
-			SMTPUserName: s.SMTP.Username,
-			SMTPPassword: s.SMTP.Password,
-			TimeOut:      s.SMTP.TimeOut,
-			StartTLS:     s.SMTP.StartTLS,
-			LoginAuth:    s.SMTP.LoginAuth,
-			TLS:          s.SMTP.TLS,
-			Charset:      "UTF-8",
-			From:         s.Auth.Email.From,
-			Subject:      s.Auth.Email.Subject,
-			ContentType:  s.Auth.Email.ContentType,
+			Host:               s.SMTP.Host,
+			Port:               s.SMTP.Port,
+			SMTPUserName:       s.SMTP.Username,
+			SMTPPassword:       s.SMTP.Password,
+			TimeOut:            s.SMTP.TimeOut,
+			StartTLS:           s.SMTP.StartTLS,
+			LoginAuth:          s.SMTP.LoginAuth,
+			TLS:                s.SMTP.TLS,
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
+			Charset:            "UTF-8",
+			From:               s.Auth.Email.From,
+			Subject:            s.Auth.Email.Subject,
+			ContentType:        s.Auth.Email.ContentType,
 		}
 		sndr := sender.NewEmailClient(params, log.Default())
 		tmpl, err := templates.Read(s.Auth.Email.MsgTemplate)
@@ -1031,16 +1121,17 @@ func (s *ServerCommand) makeNotifyDestinations(authenticator *auth.Service) ([]n
 			emailParams.AdminEmails = s.Admin.Shared.Email
 		}
 		smtpParams := ntf.SMTPParams{
-			Host:        s.SMTP.Host,
-			Port:        s.SMTP.Port,
-			TLS:         s.SMTP.TLS,
-			StartTLS:    s.SMTP.StartTLS,
-			LoginAuth:   s.SMTP.LoginAuth,
-			Username:    s.SMTP.Username,
-			Password:    s.SMTP.Password,
-			TimeOut:     s.SMTP.TimeOut,
-			ContentType: "text/html",
-			Charset:     "UTF-8",
+			Host:               s.SMTP.Host,
+			Port:               s.SMTP.Port,
+			TLS:                s.SMTP.TLS,
+			StartTLS:           s.SMTP.StartTLS,
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
+			LoginAuth:          s.SMTP.LoginAuth,
+			Username:           s.SMTP.Username,
+			Password:           s.SMTP.Password,
+			TimeOut:            s.SMTP.TimeOut,
+			ContentType:        "text/html",
+			Charset:            "UTF-8",
 		}
 		emailService, err := notify.NewEmail(emailParams, smtpParams)
 		if err != nil {
@@ -1141,7 +1232,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
-		Validator: token.ValidatorFunc(func(token string, claims token.Claims) bool { // check on each auth call (in middleware)
+		Validator: token.ValidatorFunc(func(_ string, claims token.Claims) bool { // check on each auth call (in middleware)
 			if claims.User == nil {
 				return false
 			}
@@ -1157,6 +1248,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 		Logger:            log.Default(),
 		RefreshCache:      authRefreshCache,
 		UseGravatar:       true,
+		AudSecrets:        s.Admin.RPC.SecretPerSite,
 	})
 }
 
@@ -1177,20 +1269,15 @@ func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
 
 // startTelegramAuthAndNotify initializes telegram notify and auth Telegram Bot listen loop.
 // Does nothing if telegram auth and notifications are disabled.
-// Doesn't return telegram bot username if user notifications are disabled, as that is the way frontend knows they are enabled.
-func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram, telegramBotUsername string) {
+func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram) {
 	if !contains("telegram", s.Notify.Users) && !contains("telegram", s.Notify.Admins) && !s.Auth.Telegram {
-		return nil, ""
+		return nil
 	}
 
 	var err error
 	if tg, err = s.makeTelegramNotify(); err != nil {
 		log.Printf("[WARN] failed to make telegram notify service, %s", err)
-		return nil, ""
-	}
-
-	if contains("telegram", s.Notify.Users) {
-		telegramBotUsername = tg.GetBotUsername()
+		return nil
 	}
 
 	telegramReceivers := []providers.TGUpdatesReceiver{tg}
@@ -1200,7 +1287,7 @@ func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegram
 	// start bot messages receiver for both notify and auth services
 	go providers.DispatchTelegramUpdates(ctx, tg, telegramReceivers, time.Second*5)
 
-	return tg, telegramBotUsername
+	return tg
 }
 
 // splitAtCommas split s at commas, ignoring commas in strings.
@@ -1247,11 +1334,12 @@ func splitAtCommas(s string) []string {
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	cache.LoadingCache
+	cache.LoadingCache[token.Claims]
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	expirableCache, _ := cache.NewExpirableCache(cache.TTL(5 * time.Minute))
+	o := cache.NewOpts[token.Claims]()
+	expirableCache, _ := cache.NewExpirableCache(o.TTL(5 * time.Minute))
 	return &authRefreshCache{LoadingCache: expirableCache}
 }
 
@@ -1262,5 +1350,5 @@ func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 
 // Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	_, _ = c.LoadingCache.Get(key.(string), func() (interface{}, error) { return value, nil })
+	_, _ = c.LoadingCache.Get(key.(string), func() (token.Claims, error) { return value.(token.Claims), nil })
 }

backend/app/cmd/server_test.go

@@ -79,7 +79,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	waitForHTTPServerStart(port)
 
 	providers := app.restSrv.Authenticator.Providers()
-	require.Equal(t, 9+1, len(providers), "extra auth provider")
+	require.Equal(t, 10+1, len(providers), "extra auth provider")
 	assert.Equal(t, "dev", providers[len(providers)-2].Name(), "dev auth provider")
 	// send ping
 	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
@@ -107,7 +107,7 @@ func TestServerApp_AnonMode(t *testing.T) {
 	waitForHTTPServerStart(port)
 
 	providers := app.restSrv.Authenticator.Providers()
-	require.Equal(t, 9+1, len(providers), "extra auth provider for anon")
+	require.Equal(t, 10+1, len(providers), "extra auth provider for anon")
 	assert.Equal(t, "anonymous", providers[len(providers)-1].Name(), "anon auth provider")
 
 	client := http.Client{Timeout: 10 * time.Second}
@@ -250,7 +250,7 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -290,7 +290,8 @@ func TestServerApp_WithRemote(t *testing.T) {
 	port := chooseRandomUnusedPort()
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
 		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
-		"--port=" + strconv.Itoa(port), "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+		"--port=" + strconv.Itoa(port), "--avatar.fs.path=/tmp",
+		"--admin.type=rpc", "--admin.rpc.secret_per_site", "--admin.rpc.api=http://127.0.0.1"})
 	require.NoError(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
 	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
@@ -351,6 +352,7 @@ func TestServerApp_Failed(t *testing.T) {
 	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
 	t.Log(err)
 
+	// wrong store type
 	opts = ServerCommand{}
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
 
@@ -374,6 +376,19 @@ func TestServerApp_Failed(t *testing.T) {
 			"problem subscribing to channel remark42-cache on address wrong_address: "+
 			"dial tcp: address wrong_address: missing port in address")
 	t.Log(err)
+
+	// wrong apple private key type
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+	p = flags.NewParser(&opts, flags.Default)
+	_, err = p.ParseArgs([]string{"--auth.apple.cid=123", "--auth.apple.tid=123",
+		"--auth.apple.kid=123", "--auth.apple.private-key-filepath=testdata/apple-bad.p8"})
+	assert.NoError(t, err)
+	_, err = opts.newServerApp(context.Background())
+	assert.EqualError(t, err,
+		"failed to make authenticator: an AppleProvider creating failed: "+
+			"provided private key is not ECDSA")
+	t.Log(err)
 }
 
 func TestServerApp_Shutdown(t *testing.T) {
@@ -442,6 +457,8 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
 		"--notify.telegram.token=abcd",
 		"--notify.telegram.timeout=3m",
 		"--notify.telegram.api=http://example.org",
+		"--auth.twitter.cid=123",
+		"--auth.twitter.csec=456",
 	}
 	assert.Empty(t, s.SMTP.Host)
 	assert.Empty(t, s.SMTP.Port)
@@ -467,6 +484,8 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
 			{Old: "notify.telegram.token", New: "telegram.token", Version: "1.9"},
 			{Old: "notify.telegram.timeout", New: "telegram.timeout", Version: "1.9"},
 			{Old: "notify.telegram.api", Version: "1.9"},
+			{Old: "auth.twitter.cid", Version: "1.14"},
+			{Old: "auth.twitter.csec", Version: "1.14"},
 		},
 		deprecatedFlags)
 	assert.Equal(t, "smtp.example.org", s.SMTP.Host)
@@ -597,7 +616,7 @@ func TestServerAuthHooks(t *testing.T) {
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID:   "dev",
+			ID:   "github_dev",
 			Name: "developer one",
 		},
 	}
@@ -636,14 +655,14 @@ func TestServerAuthHooks(t *testing.T) {
 	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
 
-	// block user dev as admin
+	// block user github_dev as admin
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), http.NoBody)
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/github_dev?site=remark&block=1&ttl=10d", port), http.NoBody)
 	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "user github_dev blocked")
 	b, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
@@ -711,6 +730,28 @@ func Test_splitAtCommas(t *testing.T) {
 	}
 }
 
+func Test_getAllowedDomains(t *testing.T) {
+	tbl := []struct {
+		s              ServerCommand
+		allowedDomains []string
+	}{
+		// correct example, parsed and returned as allowed domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "https://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://localhost"}}, []string{"localhost"}},
+		// incorrect URLs, so Hostname is empty but returned list doesn't include empty string as it would allow any domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "bad hostname"}}, []string{}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "not_a_hostname"}}, []string{}},
+		// test removal of 'self', multiple AllowedHosts. No deduplication is expected
+		{ServerCommand{AllowedHosts: []string{"'self'", "example.org", "test.example.org", "remark42.com"}, CommonOpts: CommonOpts{RemarkURL: "https://example.org"}}, []string{"example.org", "test.example.org", "remark42.com", "example.org"}},
+	}
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.allowedDomains, tt.s.getAllowedDomains())
+		})
+	}
+}
+
 func chooseRandomUnusedPort() (port int) {
 	for i := 0; i < 10; i++ {
 		port = 40000 + int(rand.Int31n(10000))
@@ -758,6 +799,8 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
 	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp/remark42_test", "fs", "/tmp/remark42_test", "/tmp/remark42_test"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
+	cmd.Auth.Apple.CID, cmd.Auth.Apple.KID, cmd.Auth.Apple.TID = "cid", "kid", "tid"
+	cmd.Auth.Apple.PrivateKeyFilePath = "testdata/apple.p8"
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
 	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
 	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
@@ -804,12 +847,15 @@ func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Cont
 	ctx, cancel := context.WithCancel(context.Background())
 	app, err := cmd.newServerApp(ctx)
 	require.NoError(t, err)
-
-	rand.Seed(time.Now().UnixNano())
 	return app, ctx, cancel
 }
 
 func TestMain(m *testing.M) {
 	// ignore is added only for GitHub Actions, can't reproduce locally
-	goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
+	goleak.VerifyTestMain(
+		m,
+		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/cmd/testdata/apple-bad.p8

@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKNwapOQ6rQJHetP
+HRlJBIh1OsOsUBiXb3rXXE3xpWAxAha0MH+UPRblOko+5T2JqIb+xKf9Vi3oTM3t
+KvffaOPtzKXZauscjq6NGzA3LgeiMy6q19pvkUUOlGYK6+Xfl+B7Xw6+hBMkQuGE
+nUS8nkpR5mK4ne7djIyfHFfMu4ptAgMBAAECgYA+s0PPtMq1osG9oi4xoxeAGikf
+JB3eMUptP+2DYW7mRibc+ueYKhB9lhcUoKhlQUhL8bUUFVZYakP8xD21thmQqnC4
+f63asad0ycteJMLb3r+z26LHuCyOdPg1pyLk3oQ32lVQHBCYathRMcVznxOG16VK
+I8BFfstJTaJu0lK/wQJBANYFGusBiZsJQ3utrQMVPpKmloO2++4q1v6ZR4puDQHx
+TjLjAIgrkYfwTJBLBRZxec0E7TmuVQ9uJ+wMu/+7zaUCQQDDf2xMnQqYknJoKGq+
+oAnyC66UqWC5xAnQS32mlnJ632JXA0pf9pb1SXAYExB1p9Dfqd3VAwQDwBsDDgP6
+HD8pAkEA0lscNQZC2TaGtKZk2hXkdcH1SKru/g3vWTkRHxfCAznJUaza1fx0wzdG
+GcES1Bdez0tbW4llI5By/skZc2eE3QJAFl6fOskBbGHde3Oce0F+wdZ6XIJhEgCP
+iukIcKZoZQzoiMJUoVRrA5gqnmaYDI5uRRl/y57zt6YksR3KcLUIuQJAd242M/WF
+6YAZat3q/wEeETeQq1wrooew+8lHl05/Nt0cCpV48RGEhJ83pzBm3mnwHf8lTBJH
+x6XroMXsmbnsEw==
+-----END PRIVATE KEY-----

backend/app/cmd/testdata/apple.p8

@@ -0,0 +1,6 @@
+-----BEGIN PRIVATE KEY-----
+MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgGH2MylyZjjRdauTk
+xxXW6p8VSHqIeVRRKSJPg1xn6+KgCgYIKoZIzj0DAQehRANCAAS/mNzQ7aBbIBr3
+DiHiJGIDEzi6+q3mmyhH6ZWQWFdFei2qgdyM1V6qtRPVq+yHBNSBebbR4noE/IYO
+hMdWYrKn
+-----END PRIVATE KEY-----

backend/app/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -23,7 +24,8 @@ type Opts struct {
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
 	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
-	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	// SharedSecret is only used in server command, but defined for all commands for historical reasons
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"the shared secret key used to sign JWT, should be a random, long, hard-to-guess string"`
 
 	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
@@ -54,11 +56,11 @@ func main() {
 	}
 
 	if _, err := p.Parse(); err != nil {
-		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
+		var flagsErr *flags.Error
+		if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp {
 			os.Exit(0)
-		} else {
-			os.Exit(1)
 		}
+		os.Exit(1)
 	}
 }
 
@@ -97,7 +99,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
-// nolint:gochecknoinits // can't avoid it in this place
+//nolint:gochecknoinits // can't avoid it in this place
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal, 1)

backend/app/main_test.go

@@ -64,7 +64,7 @@ func TestMain_WithWebhook(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	var webhookSent int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
 		atomic.StoreInt32(&webhookSent, 1)
 		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
 
@@ -157,5 +157,7 @@ func TestMain(m *testing.M) {
 		m,
 		goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
 		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
 	)
 }

backend/app/migrator/commento.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"time"
 
 	"github.com/umputun/remark42/backend/app/store"
@@ -110,16 +111,28 @@ func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				continue
 			}
 
+			parentID := comment.ParentHex
+			// comments with ParentHex == "root" are top-level comments
+			if parentID == "root" {
+				parentID = ""
+			}
+
+			commentURL, e := url.JoinPath("https://", comment.Domain, comment.Path)
+			if e != nil {
+				log.Printf("[WARN] can't construct comment URL in commento import, %s", err.Error())
+			}
+			log.Printf("[ERROR] commentoURL: %s", commentURL)
+
 			c := store.Comment{
 				ID: comment.CommentHex,
 				Locator: store.Locator{
-					URL:    comment.Path,
+					URL:    commentURL,
 					SiteID: siteID,
 				},
 				User:      u,
 				Text:      comment.Markdown,
 				Timestamp: comment.CreationDate,
-				ParentID:  comment.ParentHex,
+				ParentID:  parentID,
 				Imported:  true,
 			}
 

backend/app/migrator/native_test.go

@@ -163,7 +163,7 @@ func TestNative_ImportManyWithError(t *testing.T) {
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
 	for i := 0; i < 100; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
+		fmt.Fprintf(buf, goodRec, i)
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")

backend/app/migrator/testdata/commento.json

@@ -4,7 +4,7 @@
 	{
 	  "commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "anonymous",
 	  "markdown": "Example comment created by user.",
 	  "html": "",
@@ -18,7 +18,7 @@
 	{
 	  "commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 	  "markdown": "Example 2 comment created by user.",
 	  "html": "",
@@ -32,7 +32,7 @@
 	{
 	  "commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
 	  "markdown": "Great reply!",
 	  "html": "",

backend/app/migrator/wordpress.go

@@ -16,7 +16,8 @@ const wpTimeLayout = "2006-01-02 15:04:05"
 
 // WordPress implements Importer from WP xml
 type WordPress struct {
-	DataStore Store
+	DataStore                  Store
+	DisableFancyTextFormatting bool
 }
 
 type wpItem struct {
@@ -138,7 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								ParentID:  comment.PID,
 								Imported:  true,
 							}
-							commentsCh <- commentFormatter.Format(c)
+							commentsCh <- commentFormatter.Format(c, w.DisableFancyTextFormatting)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
 								log.Printf("[DEBUG] processed %d comments", stats.inpComments)

backend/app/migrator/wordpress_test.go

@@ -24,7 +24,7 @@ func TestWordPress_Import(t *testing.T) {
 
 	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 	defer dataStore.Close()
-	wp := WordPress{DataStore: &dataStore}
+	wp := WordPress{DataStore: &dataStore, DisableFancyTextFormatting: false}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
@@ -41,7 +41,7 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
-	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.Equal(t, "<p>«Mekkatorque» was over in that tent up to the right</p>\n", c.Text)
 	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
@@ -54,6 +54,18 @@ func TestWordPress_Import(t *testing.T) {
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
+
+	// test with DisableFancyTextFormatting
+	wp = WordPress{DataStore: &dataStore, DisableFancyTextFormatting: true}
+	size, err = wp.Import(strings.NewReader(xmlTestWP), siteID)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, size)
+
+	last, err = dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
+
+	assert.Equal(t, "<p>&#34;Mekkatorque&#34; was over in that tent up to the right</p>\n", last[0].Text)
 }
 
 func TestWordPress_Convert(t *testing.T) {
@@ -247,7 +259,7 @@ var xmlTestWP = `
 			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
 			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
 			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
-			<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
+			<wp:comment_content><![CDATA["Mekkatorque" was over in that tent up to the right]]></wp:comment_content>
 			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
 			<wp:comment_type><![CDATA[]]></wp:comment_type>
 			<wp:comment_parent>13</wp:comment_parent>

backend/app/notify/notify.go

@@ -143,6 +143,12 @@ func (s *Service) SubmitVerification(req VerificationRequest) {
 // Close queue channel and wait for completion
 func (s *Service) Close() {
 	if s.queue != nil {
+		// don't panic in case service is already closed
+		select {
+		case <-s.ctx.Done():
+			return
+		default:
+		}
 		log.Print("[DEBUG] close notifier")
 		close(s.queue)
 		close(s.verificationQueue)

backend/app/notify/notify_test.go

@@ -21,6 +21,8 @@ func TestService_NoDestinations(t *testing.T) {
 	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Submit(Request{Comment: store.Comment{ID: "123"}})
 	s.Close()
+	// second call should not result in panic
+	s.Close()
 }
 
 func TestService_WithDestinations(t *testing.T) {

backend/app/notify/webhook.go

@@ -3,6 +3,7 @@ package notify
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"text/template"
 	"time"
@@ -12,7 +13,7 @@ import (
 )
 
 const (
-	webhookDefaultTemplate = `{"text": "{{.Text}}"}`
+	webhookDefaultTemplate = `{"text": {{.Text | escapeJSONString}}}`
 )
 
 // WebhookParams contain settings for webhook notifications
@@ -49,7 +50,7 @@ func NewWebhook(params WebhookParams) (*Webhook, error) {
 		params.Template = webhookDefaultTemplate
 	}
 
-	payloadTmpl, err := template.New("webhook").Parse(params.Template)
+	payloadTmpl, err := template.New("webhook").Funcs(template.FuncMap{"escapeJSONString": escapeJSONString}).Parse(params.Template)
 	if err != nil {
 		return nil, fmt.Errorf("unable to parse webhook template: %w", err)
 	}
@@ -82,3 +83,12 @@ func (w *Webhook) SendVerification(_ context.Context, _ VerificationRequest) err
 func (w *Webhook) String() string {
 	return fmt.Sprintf("%s to %s", w.Webhook.String(), w.url)
 }
+
+// escapeJSONString escapes string for JSON
+func escapeJSONString(s string) (string, error) {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

backend/app/notify/webhook_test.go

@@ -2,6 +2,9 @@ package notify
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -34,6 +37,32 @@ func TestWebhook_NewWebhook(t *testing.T) {
 	assert.Contains(t, err.Error(), "unable to parse webhook template")
 }
 
+// https://github.com/umputun/remark42/issues/1791
+func TestWebhook_ReceiveValidJSON(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/webhook-notify")
+		assert.Equal(t, "POST", r.Method)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		t.Log("received body", string(body))
+		assert.JSONEq(t, `{"text": "<p>testme</p>\n"}`, string(body))
+	}))
+	defer ts.Close()
+
+	wh, err := NewWebhook(WebhookParams{
+		URL:     ts.URL + "/webhook-notify",
+		Headers: []string{"Content-Type:application/json,text/plain"},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	f := store.NewCommentFormatter()
+	c := store.Comment{Text: f.FormatText("testme", false), ParentID: "1", ID: "999"}
+
+	err = wh.Send(context.Background(), Request{Comment: c})
+	assert.NoError(t, err)
+}
+
 func TestWebhook_Send(t *testing.T) {
 	wh, err := NewWebhook(WebhookParams{
 		URL:     "bad-url",

backend/app/rest/api/admin.go

@@ -9,7 +9,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 

backend/app/rest/api/admin_test.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
 	"github.com/stretchr/testify/assert"
@@ -111,7 +111,7 @@ func TestAdmin_Title(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"127.0.0.1"})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
 			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
@@ -348,7 +348,7 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
 	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
-	srv.pubRest.cache = cache.NewScache(cache.NewNopCache()) // TODO: with lru cache it won't be refreshed and invalidated for long
+	srv.pubRest.cache = cache.NewScache[[]byte](cache.NewNopCache[[]byte]()) // TODO: with lru cache it won't be refreshed and invalidated for long
 	// time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
@@ -513,6 +513,7 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.Error(t, err)
 
+	// test format "tree"
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	comments := commentsWithInfo{}
@@ -521,6 +522,16 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 	assert.True(t, comments.Info.ReadOnly)
 	t.Logf("%+v", comments)
+
+	// test format "plain"
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah")
+	assert.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
 }
 
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
@@ -743,9 +754,9 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "provider1_user1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "provider1_user2"}}
 
 	_, err := srv.DataService.Create(c1)
 	assert.NoError(t, err)
@@ -768,13 +779,13 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
-			Id:        "1234567",
+			Id:        "provider1_1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID: "user1",
+			ID: "provider1_user1",
 			Attributes: map[string]interface{}{
 				"delete_me": true,
 			},

backend/app/rest/api/migrator.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 
@@ -136,7 +136,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
 		w.Header().Set("Content-Type", "application/gzip")
 		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
 		gzWriter := gzip.NewWriter(w)
 		defer func() {
 			if e := gzWriter.Close(); e != nil {
@@ -163,7 +162,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
 		return
 	}
-	defer r.Body.Close()
+	defer r.Body.Close() //nolint gosec // we don't care about response body
 
 	// start remap procedure with mapper
 	go func() {

backend/app/rest/api/migrator_test.go

@@ -9,6 +9,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -49,6 +50,22 @@ func TestMigrator_Import(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
@@ -84,6 +101,22 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
@@ -108,6 +141,22 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 3, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments), "2 comments with 1 reply")
 }
 
 func TestMigrator_ImportFromCommento(t *testing.T) {
@@ -115,7 +164,7 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
-"domain":"example.com","url":"https://example.com/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
+"domain":"example.com","url":"/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z",
 "direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
@@ -137,6 +186,63 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+}
+
+func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	r, err := os.Open("testdata/commento.json")
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
+	assert.NoError(t, err)
+	req.Header.Add("Content-Type", "application/json; charset=utf-8")
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+	assert.NoError(t, resp.Body.Close())
+
+	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 7, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 5, len(comments.Comments), "five comments with two replies")
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
@@ -197,6 +303,20 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
@@ -236,6 +356,14 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_Export(t *testing.T) {
@@ -263,6 +391,16 @@ func TestMigrator_Export(t *testing.T) {
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 
+	// export wrong site, should result in error
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=test", http.NoBody)
+	require.NoError(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+
 	// check file mode
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", http.NoBody)
 	require.NoError(t, err)
@@ -339,6 +477,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
@@ -347,6 +486,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// we want remap urls to another domain - www.remark42.com
@@ -364,6 +504,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
@@ -372,6 +522,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+	require.True(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// should find nothing from previous url
@@ -381,6 +541,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
 	require.Equal(t, http.StatusOK, code)
@@ -388,6 +549,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_RemapReject(t *testing.T) {

backend/app/rest/api/rest.go

@@ -22,7 +22,7 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/logger"
@@ -59,26 +59,29 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter       float64
-	EmailNotifications  bool
-	TelegramBotUsername string
-	EmojiEnabled        bool
-	SimpleView          bool
-	ProxyCORS           bool
-	SendJWTHeader       bool
-	AllowedAncestors    []string // sets Content-Security-Policy "frame-ancestors ..."
-	SubscribersOnly     bool
-	DisableSignature    bool // prevent signature from being added to headers
+	UpdateLimiter              float64
+	EmailNotifications         bool
+	TelegramNotifications      bool
+	EmojiEnabled               bool
+	SimpleView                 bool
+	ProxyCORS                  bool
+	SendJWTHeader              bool
+	AllowedAncestors           []string // sets Content-Security-Policy "frame-ancestors ..."
+	SubscribersOnly            bool
+	DisableSignature           bool // prevent signature from being added to headers
+	DisableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
+	ExternalImageProxy         bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	pubRest   public
-	privRest  private
-	adminRest admin
-	rssRest   rss
+	pubRest          public
+	privRest         private
+	adminRest        admin
+	rssRest          rss
+	openRouteLimiter float64
 }
 
 // LoadingCache defines interface for caching
@@ -89,7 +92,7 @@ type LoadingCache interface {
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
-
+const openRouteLimiter = 10     // limit for open routes
 const lastCommentsScope = "last"
 
 type commentsWithInfo struct {
@@ -97,6 +100,11 @@ type commentsWithInfo struct {
 	Info     store.PostInfo  `json:"info,omitempty"`
 }
 
+type treeWithInfo struct {
+	*service.Tree
+	Info store.PostInfo `json:"info,omitempty"`
+}
+
 // Run the lister and request's router, activate rest server
 func (s *Rest) Run(address string, port int) {
 	if address == "*" {
@@ -192,8 +200,13 @@ func (s *Rest) makeHTTPServer(address string, port int, router http.Handler) *ht
 }
 
 func (s *Rest) routes() chi.Router {
+	if s.openRouteLimiter == 0 {
+		// set the default open route limiter. Just a safety measure as it should be set by Run method anyway
+		s.openRouteLimiter = openRouteLimiter
+	}
 	router := chi.NewRouter()
 	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(securityHeadersMiddleware(s.ExternalImageProxy, s.AllowedAncestors))
 	if !s.DisableSignature {
 		router.Use(R.AppInfo("remark42", "umputun", s.Version))
 	}
@@ -215,11 +228,6 @@ func (s *Rest) routes() chi.Router {
 		router.Use(corsMiddleware.Handler)
 	}
 
-	if len(s.AllowedAncestors) > 0 {
-		log.Printf("[INFO] allowed from %+v only", s.AllowedAncestors)
-		router.Use(frameAncestors(s.AllowedAncestors))
-	}
-
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
 	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
@@ -251,7 +259,7 @@ func (s *Rest) routes() chi.Router {
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(middleware.Timeout(30 * time.Second))
-			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.openRouteLimiter, nil)))
 			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/config", s.configCtrl)
 			ropen.Get("/find", s.pubRest.findCommentsCtrl)
@@ -369,16 +377,17 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 	}
 
 	privGrp := private{
-		dataService:      s.DataService,
-		cache:            s.Cache,
-		imageService:     s.ImageService,
-		commentFormatter: s.CommentFormatter,
-		readOnlyAge:      s.ReadOnlyAge,
-		authenticator:    s.Authenticator,
-		notifyService:    s.NotifyService,
-		telegramService:  s.TelegramService,
-		remarkURL:        s.RemarkURL,
-		anonVote:         s.AnonVote,
+		dataService:                s.DataService,
+		cache:                      s.Cache,
+		imageService:               s.ImageService,
+		commentFormatter:           s.CommentFormatter,
+		readOnlyAge:                s.ReadOnlyAge,
+		authenticator:              s.Authenticator,
+		notifyService:              s.NotifyService,
+		telegramService:            s.TelegramService,
+		remarkURL:                  s.RemarkURL,
+		anonVote:                   s.AnonVote,
+		disableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	admGrp := admin{
@@ -414,44 +423,46 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 	emails, _ := s.DataService.AdminStore.Email(siteID)
 
 	cnf := struct {
-		Version             string   `json:"version"`
-		EditDuration        int      `json:"edit_duration"`
-		AdminEdit           bool     `json:"admin_edit"`
-		MaxCommentSize      int      `json:"max_comment_size"`
-		Admins              []string `json:"admins"`
-		AdminEmail          string   `json:"admin_email"`
-		Auth                []string `json:"auth_providers"`
-		AnonVote            bool     `json:"anon_vote"`
-		LowScore            int      `json:"low_score"`
-		CriticalScore       int      `json:"critical_score"`
-		PositiveScore       bool     `json:"positive_score"`
-		ReadOnlyAge         int      `json:"readonly_age"`
-		MaxImageSize        int      `json:"max_image_size"`
-		EmailNotifications  bool     `json:"email_notifications"`
-		TelegramBotUsername string   `json:"telegram_bot_username"`
-		EmojiEnabled        bool     `json:"emoji_enabled"`
-		SimpleView          bool     `json:"simple_view"`
-		SendJWTHeader       bool     `json:"send_jwt_header"`
-		SubscribersOnly     bool     `json:"subscribers_only"`
+		Version               string   `json:"version"`
+		EditDuration          int      `json:"edit_duration"`
+		AdminEdit             bool     `json:"admin_edit"`
+		MinCommentSize        int      `json:"min_comment_size"`
+		MaxCommentSize        int      `json:"max_comment_size"`
+		Admins                []string `json:"admins"`
+		AdminEmail            string   `json:"admin_email"`
+		Auth                  []string `json:"auth_providers"`
+		AnonVote              bool     `json:"anon_vote"`
+		LowScore              int      `json:"low_score"`
+		CriticalScore         int      `json:"critical_score"`
+		PositiveScore         bool     `json:"positive_score"`
+		ReadOnlyAge           int      `json:"readonly_age"`
+		MaxImageSize          int      `json:"max_image_size"`
+		EmailNotifications    bool     `json:"email_notifications"`
+		TelegramNotifications bool     `json:"telegram_notifications"`
+		EmojiEnabled          bool     `json:"emoji_enabled"`
+		SimpleView            bool     `json:"simple_view"`
+		SendJWTHeader         bool     `json:"send_jwt_header"`
+		SubscribersOnly       bool     `json:"subscribers_only"`
 	}{
-		Version:             s.Version,
-		EditDuration:        int(s.DataService.EditDuration.Seconds()),
-		AdminEdit:           s.DataService.AdminEdits,
-		MaxCommentSize:      s.DataService.MaxCommentSize,
-		Admins:              admins,
-		AdminEmail:          emails,
-		LowScore:            s.ScoreThresholds.Low,
-		CriticalScore:       s.ScoreThresholds.Critical,
-		PositiveScore:       s.DataService.PositiveScore,
-		ReadOnlyAge:         s.ReadOnlyAge,
-		MaxImageSize:        s.ImageService.MaxSize,
-		EmailNotifications:  s.EmailNotifications,
-		TelegramBotUsername: s.TelegramBotUsername,
-		EmojiEnabled:        s.EmojiEnabled,
-		AnonVote:            s.AnonVote,
-		SimpleView:          s.SimpleView,
-		SendJWTHeader:       s.SendJWTHeader,
-		SubscribersOnly:     s.SubscribersOnly,
+		Version:               s.Version,
+		EditDuration:          int(s.DataService.EditDuration.Seconds()),
+		AdminEdit:             s.DataService.AdminEdits,
+		MinCommentSize:        s.DataService.MinCommentSize,
+		MaxCommentSize:        s.DataService.MaxCommentSize,
+		Admins:                admins,
+		AdminEmail:            emails,
+		LowScore:              s.ScoreThresholds.Low,
+		CriticalScore:         s.ScoreThresholds.Critical,
+		PositiveScore:         s.DataService.PositiveScore,
+		ReadOnlyAge:           s.ReadOnlyAge,
+		MaxImageSize:          s.ImageService.MaxSize,
+		EmailNotifications:    s.EmailNotifications,
+		TelegramNotifications: s.TelegramNotifications,
+		EmojiEnabled:          s.EmojiEnabled,
+		AnonVote:              s.AnonVote,
+		SimpleView:            s.SimpleView,
+		SendJWTHeader:         s.SendJWTHeader,
+		SubscribersOnly:       s.SubscribersOnly,
 	}
 
 	cnf.Auth = []string{}
@@ -609,19 +620,23 @@ func cacheControl(expiration time.Duration, version string) func(http.Handler) h
 	}
 }
 
-// frameAncestors is a middleware setting Content-Security-Policy "frame-ancestors host1 host2 ..."
-// prevents loading of comments widgets from any other origins. In case if the list of allowed empty, ignored.
-func frameAncestors(hosts []string) func(http.Handler) http.Handler {
-	return func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-			if len(hosts) == 0 {
-				h.ServeHTTP(w, r)
-				return
+// securityHeadersMiddleware sets security-related headers: Content-Security-Policy and Permissions-Policy
+func securityHeadersMiddleware(imageProxyEnabled bool, allowedAncestors []string) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			imgSrc := "*"
+			if imageProxyEnabled {
+				imgSrc = "'self'"
 			}
-			w.Header().Set("Content-Security-Policy", "frame-ancestors "+strings.Join(hosts, " ")+";")
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
+			frameAncestors := "*"
+			if len(allowedAncestors) > 0 {
+				log.Printf("[INFO] frame embedding allowed from %+v only", allowedAncestors)
+				frameAncestors = strings.Join(allowedAncestors, " ")
+			}
+			w.Header().Set("Content-Security-Policy", fmt.Sprintf("default-src 'none'; base-uri 'none'; form-action 'none'; connect-src 'self'; frame-src 'self'; img-src %s; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; font-src data:; object-src 'none'; frame-ancestors %s;", imgSrc, frameAncestors))
+			w.Header().Set("Permissions-Policy", "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(), clipboard-write=(), gamepad=(), hid=(), idle-detection=(), interest-cohort=(), serial=(), unload=(), window-management=()")
+			next.ServeHTTP(w, r)
+		})
 	}
 }
 

backend/app/rest/api/rest_private.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"crypto/sha1" //nolint:gosec //not used for security
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"io"
@@ -17,7 +18,7 @@ import (
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
@@ -33,16 +34,17 @@ import (
 )
 
 type private struct {
-	dataService      privStore
-	cache            LoadingCache
-	readOnlyAge      int
-	commentFormatter *store.CommentFormatter
-	imageService     *image.Service
-	notifyService    *notify.Service
-	authenticator    *auth.Service
-	telegramService  telegramService
-	remarkURL        string
-	anonVote         bool
+	dataService                privStore
+	cache                      LoadingCache
+	readOnlyAge                int
+	commentFormatter           *store.CommentFormatter
+	imageService               *image.Service
+	notifyService              *notify.Service
+	authenticator              *auth.Service
+	telegramService            telegramService
+	remarkURL                  string
+	anonVote                   bool
+	disableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 }
 
 // telegramService is a subset of Telegram service used for setting up user telegram notifications
@@ -87,14 +89,14 @@ func (s *private) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 	comment.Sanitize()
 
-	// check if images are valid
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	// check if images are valid, omit proxied images as they are lazy-loaded
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		err := s.imageService.ResetCleanupTimer(id)
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't renew staged picture cleanup timer", rest.ErrImgNotFound)
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
 			return
 		}
 	}
@@ -127,10 +129,10 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 
-	// check if images are valid
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	// check if images are valid, omit proxied images as they are lazy-loaded
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		_, err := s.imageService.Load(id)
 		if err != nil {
 			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
@@ -150,7 +152,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	id, err := s.dataService.Create(comment)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentRestrictWords)
 		return
 	}
@@ -187,7 +189,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't read comment details from body", rest.ErrDecode)
 		return
 	}
 
@@ -211,7 +213,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.commentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text, s.disableFancyTextFormatting),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
@@ -219,7 +221,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res, err := s.dataService.EditComment(locator, id, editReq)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
@@ -244,7 +246,7 @@ func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
 		}
-		if len(email) > 0 {
+		if email != "" {
 			user.EmailSubscription = true
 		}
 	}
@@ -307,28 +309,59 @@ func (s *private) getEmailCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // sendEmailConfirmationCtrl gets address and siteID from query, makes confirmation token and sends it to user.
-// GET /email/subscribe?site=siteID&address=someone@example.com
+// In case user is logged in with the same email, and auto_confirm is true, confirm it right away.
+// In case of quick confirmation, "updated" is set to true, otherwise - to false.
+// POST /email/subscribe with site and address in json body
+//
 //nolint:dupl // too hard to deduplicate that logic, as then it's tricky to use SendErrorJSON
 func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
-	address := r.URL.Query().Get("address")
-	siteID := r.URL.Query().Get("site")
-	if address == "" {
+
+	subscribe := struct {
+		Site        string
+		Address     string
+		autoConfirm bool
+	}{autoConfirm: true}
+	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &subscribe); err != nil {
+		if err != io.EOF {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse request body", rest.ErrDecode)
+			return
+		}
+		// old behavior fallback, reading from the query params. Auto confirm is false in this case.
+		subscribe.Address = r.URL.Query().Get("address")
+		subscribe.Site = r.URL.Query().Get("site")
+		subscribe.autoConfirm = false
+	}
+
+	if subscribe.Address == "" {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest,
 			fmt.Errorf("missing parameter"), "address parameter is required", rest.ErrInternal)
 		return
 	}
-	existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
-	if err != nil {
-		log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
+	existingAddress, getErr := s.dataService.GetUserEmail(subscribe.Site, user.ID)
+	if getErr != nil {
+		log.Printf("[WARN] can't read email for %s, %v", user.ID, getErr)
 	}
-	if address == existingAddress {
+	if subscribe.Address == existingAddress {
 		rest.SendErrorJSON(w, r, http.StatusConflict,
 			fmt.Errorf("already verified"), "email address is already verified for this user", rest.ErrInternal)
 		return
 	}
+
+	// in case the user logged in with the same email as they try to subscribe with, confirm it right away
+	// this behavior is different from the previous one and is hidden behind the autoConfirm flag,
+	// which is true for the new API, and false for the old one
+	//
+	// nolint:gosec // this is not used for security purposes
+	if subscribe.autoConfirm &&
+		strings.HasPrefix(user.ID, "email_") &&
+		strings.TrimPrefix(user.ID, "email_") == token.HashID(sha1.New(), subscribe.Address) {
+		s.setEmail(w, r, user.ID, subscribe.Site, subscribe.Address)
+		return
+	}
+
 	claims := token.Claims{
-		Handshake: &token.Handshake{ID: user.ID + "::" + address},
+		Handshake: &token.Handshake{ID: user.ID + "::" + subscribe.Address},
 		StandardClaims: jwt.StandardClaims{
 			Audience:  r.URL.Query().Get("site"),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
@@ -345,14 +378,14 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
 
 	s.notifyService.SubmitVerification(
 		notify.VerificationRequest{
-			SiteID: siteID,
+			SiteID: subscribe.Site,
 			User:   user.Name,
-			Email:  address,
+			Email:  subscribe.Address,
 			Token:  tkn,
 		},
 	)
 
-	render.JSON(w, r, R.JSON{"user": user, "address": address})
+	render.JSON(w, r, R.JSON{"user": user, "address": subscribe.Address, "updated": false})
 }
 
 // telegramSubscribeCtrl generates and verifies telegram notification request
@@ -399,7 +432,7 @@ func (s *private) telegramSubscribeCtrl(w http.ResponseWriter, r *http.Request)
 	var address, siteID string
 	address, siteID, err := s.telegramService.CheckToken(queryToken, user.ID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set telegram for user", rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusNotFound, err, "request is not verified yet", rest.ErrInternal)
 		return
 	}
 
@@ -416,17 +449,29 @@ func (s *private) telegramSubscribeCtrl(w http.ResponseWriter, r *http.Request)
 }
 
 // setConfirmedEmailCtrl uses provided token parameter (generated by sendEmailConfirmationCtrl) to set email and add it to user token
-// PUT /email/confirm?site=siteID&tkn=jwt
+// POST /email/confirm with site and token in json body
 func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) {
-	tkn := r.URL.Query().Get("tkn")
-	if tkn == "" {
+	user := rest.MustGetUserInfo(r)
+
+	confirm := struct {
+		Site  string
+		Token string
+	}{}
+	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &confirm); err != nil {
+		if err != io.EOF {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse request body", rest.ErrDecode)
+			return
+		}
+		// old behavior fallback, reading from the query params
+		confirm.Token = r.URL.Query().Get("tkn")
+		confirm.Site = r.URL.Query().Get("site")
+	}
+
+	if confirm.Token == "" {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal)
 		return
 	}
-	user := rest.MustGetUserInfo(r)
-	siteID := r.URL.Query().Get("site")
-
-	confClaims, err := s.authenticator.TokenService().Parse(tkn)
+	confClaims, err := s.authenticator.TokenService().Parse(confirm.Token)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
 		return
@@ -444,17 +489,20 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	address := elems[1]
+	s.setEmail(w, r, user.ID, confirm.Site, address)
+}
 
-	log.Printf("[DEBUG] set email for user %s", user.ID)
+func (s *private) setEmail(w http.ResponseWriter, r *http.Request, userID, siteID, address string) {
+	log.Printf("[DEBUG] set email for user %s", userID)
 
-	val, err := s.dataService.SetUserEmail(siteID, user.ID, address)
+	val, err := s.dataService.SetUserEmail(siteID, userID, address)
 	if err != nil {
 		code := parseError(err, rest.ErrInternal)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set email for user", code)
 		return
 	}
 
-	// update User.Email from the token
+	// update User.Email field
 	claims, _, err := s.authenticator.TokenService().Get(r)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)

backend/app/rest/api/rest_private_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
@@ -24,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/image"
 )
@@ -69,7 +71,7 @@ func TestRest_CreateFilteredCode(t *testing.T) {
 
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.NoError(t, err)
+	require.NoError(t, err, string(b))
 	loc := c["locator"].(map[string]interface{})
 	assert.Equal(t, "remark42", loc["site"])
 	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
@@ -79,6 +81,86 @@ func TestRest_CreateFilteredCode(t *testing.T) {
 	assert.True(t, len(c["id"].(string)) > 8)
 }
 
+// based on issue https://github.com/umputun/remark42/issues/1631
+func TestRest_CreateAndPreviewWithImage(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	ts.Close()
+	defer teardown()
+
+	srv.ImageService.ProxyAPI = srv.RemarkURL + "/api/v1/img"
+	srv.ImageProxy = &proxy.Image{
+		HTTP2HTTPS:    true,
+		CacheExternal: true,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     srv.RemarkURL,
+		ImageService:  srv.ImageService,
+	}
+	srv.CommentFormatter = store.NewCommentFormatter(srv.ImageProxy)
+	// need to recreate the server with new ImageProxy, otherwise old one will be used
+	ts = httptest.NewServer(srv.routes())
+	defer ts.Close()
+
+	var pngRead bool
+	// server with the test PNG image
+	pngServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, e := io.Copy(w, gopherPNG())
+		assert.NoError(t, e)
+		pngRead = true
+	}))
+	defer pngServer.Close()
+
+	t.Run("create", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/comment",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		c := R.JSON{}
+		err = json.Unmarshal(b, &c)
+		require.NoError(t, err, string(b))
+		assert.NotContains(t, c["text"], pngServer.URL)
+		assert.Contains(t, c["text"], srv.RemarkURL)
+		loc := c["locator"].(map[string]interface{})
+		assert.Equal(t, "remark42", loc["site"])
+		assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
+		assert.True(t, len(c["id"].(string)) > 8)
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+	})
+
+	t.Run("preview", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/preview",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		assert.NotContains(t, string(b), pngServer.URL)
+		assert.Contains(t, string(b), srv.RemarkURL)
+
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+		// retrieve the image from the cache
+		imgURL := strings.Split(strings.Split(string(b), "src=\"")[1], "\"")[0]
+		// replace srv.RemarkURL with ts.URL
+		imgURL = strings.ReplaceAll(imgURL, srv.RemarkURL, ts.URL)
+		resp, err = http.Get(imgURL)
+		assert.NoError(t, err)
+		b, err = io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+		// compare image to original gopher png after decoding from base64
+		assert.Equal(t, gopher, base64.StdEncoding.EncodeToString(b))
+
+		assert.Equal(t, true, pngRead, "original image accessed to be shown to user")
+	})
+
+}
+
 func TestRest_CreateOldPost(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -166,6 +248,25 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
 	assert.Equal(t, "invalid comment", c["details"])
 }
 
+func TestRest_CreateRelativeURL(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// check that it's not possible to click insert URL button and not alter the URL in it (which is `url` by default)
+	relativeURLText := `{"text": "here is a link with relative URL: [google.com](url)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
+	resp, err := post(t, ts.URL+"/api/v1/comment", relativeURLText)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	assert.Equal(t, "links should start with mailto:, http:// or https://", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_CreateRejected(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -267,6 +368,56 @@ func TestRest_CreateAndGet(t *testing.T) {
 	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false, IP: ""}, comment.User, "no ip")
 }
 
+func TestRest_CreateWithQuotes(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create comment with quotes with smartypants
+	resp, err := post(t, ts.URL+"/api/v1/comment",
+		`{"text": "smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id := c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment := store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>smartpants «quoted» text</p>\n", comment.Text)
+	assert.Equal(t, "smartpants \"quoted\" text", comment.Orig)
+
+	// create comment with quotes without smartypants
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/comment",
+		`{"text": "no_smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c = R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id = c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment = store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>no_smartpants &#34;quoted&#34; text</p>\n", comment.Text)
+	assert.Equal(t, "no_smartpants \"quoted\" text", comment.Orig)
+}
+
 func TestRest_Update(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -574,7 +725,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=remark42&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), http.NoBody)
 		assert.NoError(t, err)
-		req.Header.Add("X-JWT", devToken)
+		req.Header.Add("X-JWT", dev2Token)
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
 		assert.NoError(t, resp.Body.Close())
@@ -583,7 +734,7 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, http.StatusOK, vote(1), "first vote allowed")
 	assert.Equal(t, http.StatusBadRequest, vote(1), "second vote rejected")
-	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
@@ -594,7 +745,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -603,7 +754,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, 0, cr.Vote)
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed one more time")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -612,7 +763,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, -1, cr.Vote)
 
 	assert.Equal(t, http.StatusBadRequest, vote(-1), "dbl vote not allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -709,7 +860,7 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 
 	// issue good token
 	claims := token.Claims{
-		Handshake: &token.Handshake{ID: "dev::good@example.com"},
+		Handshake: &token.Handshake{ID: "provider1_dev::good@example.com"},
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
 			ExpiresAt: time.Now().Add(10 * time.Minute).Unix(),
@@ -728,19 +879,25 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 		responseCode int
 		noAuth       bool
 		cookieEmail  string
+		body         string
 	}{
 		{description: "issue delete request without auth", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusUnauthorized, noAuth: true},
 		{description: "issue delete request without site_id", url: "/api/v1/email", method: http.MethodDelete, responseCode: http.StatusBadRequest},
 		{description: "delete non-existent user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
-		{description: "set user email, token not set", url: "/api/v1/email/confirm?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
-		{description: "send email confirmation without address", url: "/api/v1/email/subscribe?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
-		{description: "send email confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
-		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "set user email, token not set", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusBadRequest, body: `{"site":"remark42"}`},
+		{description: "set user email, token not set, old query param", url: "/api/v1/email/confirm?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send email confirmation without address", url: "/api/v1/email/subscribe", method: http.MethodPost, responseCode: http.StatusBadRequest, body: `{"site":"remark42"}`},
+		{description: "send email confirmation without address, old query param", url: "/api/v1/email/subscribe?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
+		{description: "send email confirmation", url: "/api/v1/email/subscribe", method: http.MethodPost, responseCode: http.StatusOK, body: `{"site":"remark42","address":"good@example.com"}`},
+		{description: "send email confirmation, old query param", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
+		{description: "set user email, token is good", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com", body: fmt.Sprintf(`{"site":"remark42","token":%q}`, goodToken)},
+		{description: "set user email, token is good, old query param", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
 		{description: "send confirmation with same address", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusConflict},
 		{description: "get user email", url: "/api/v1/email?site=remark42", method: http.MethodGet, responseCode: http.StatusOK},
 		{description: "delete user email", url: "/api/v1/email?site=remark42", method: http.MethodDelete, responseCode: http.StatusOK},
 		{description: "send another confirmation", url: "/api/v1/email/subscribe?site=remark42&address=good@example.com", method: http.MethodPost, responseCode: http.StatusOK},
-		{description: "set user email, token is good", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
+		{description: "set user email, token is good", url: "/api/v1/email/confirm", method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com", body: fmt.Sprintf(`{"site":"remark42","token":%q}`, goodToken)},
+		{description: "set user email, token is good, old query param", url: fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK, cookieEmail: "good@example.com"},
 		{description: "unsubscribe user, no token", url: "/email/unsubscribe.html?site=remark42", method: http.MethodPost, responseCode: http.StatusBadRequest},
 		{description: "unsubscribe user, wrong token", url: "/email/unsubscribe.html?site=remark42&tkn=jwt", method: http.MethodGet, responseCode: http.StatusForbidden},
 		{description: "unsubscribe user, good token", url: fmt.Sprintf("/email/unsubscribe.html?site=remark42&tkn=%s", goodToken), method: http.MethodPost, responseCode: http.StatusOK},
@@ -761,7 +918,11 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 	for _, x := range testData {
 		x := x
 		t.Run(x.description, func(t *testing.T) {
-			req, err := http.NewRequest(x.method, ts.URL+x.url, http.NoBody)
+			reqBody := io.NopCloser(strings.NewReader(x.body))
+			if x.body == "" {
+				reqBody = http.NoBody
+			}
+			req, err := http.NewRequest(x.method, ts.URL+x.url, reqBody)
 			require.NoError(t, err)
 			if !x.noAuth {
 				req.Header.Add("X-JWT", devToken)
@@ -798,7 +959,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -837,7 +998,11 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.Empty(t, mockDestination.Get()[1].Emails)
 
 	// send confirmation token for email
-	req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe",
+		io.NopCloser(strings.NewReader(`{"site": "remark42", "address": "good@example.com"}`)),
+	)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -852,8 +1017,31 @@ func TestRest_EmailNotification(t *testing.T) {
 	assert.Equal(t, "good@example.com", mockDestination.GetVerify()[0].Email)
 	verificationToken := mockDestination.GetVerify()[0].Token
 
+	// get user information to verify lack of the subscription
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var clearUser store.User
+	err = json.Unmarshal(body, &clearUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: false,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, clearUser)
+
 	// verify email
-	req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/confirm",
+		io.NopCloser(strings.NewReader(fmt.Sprintf(`{"site": "remark42", "token": %q}`, verificationToken))),
+	)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -864,7 +1052,10 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
 
 	// get user information to verify the subscription
-	req, err = http.NewRequest(http.MethodGet, ts.URL+"/api/v1/user?site=remark42", http.NoBody)
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
 	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	resp, err = client.Do(req)
@@ -873,11 +1064,11 @@ func TestRest_EmailNotification(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
-	var user store.User
-	err = json.Unmarshal(body, &user)
+	var subscribedUser store.User
+	err = json.Unmarshal(body, &subscribedUser)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", EmailSubscription: true,
-		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: true,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, subscribedUser)
 
 	// create child comment from another user, email notification expected
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
@@ -928,6 +1119,80 @@ func TestRest_EmailNotification(t *testing.T) {
 	time.Sleep(time.Millisecond * 30)
 	require.Equal(t, 4, len(mockDestination.Get()))
 	assert.Empty(t, mockDestination.Get()[3].Emails)
+
+	// confirm email via subscribe call with query params, old behavior, email notification is expected
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com",
+		http.NoBody,
+	)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 2, len(mockDestination.GetVerify()), "verification email was sent")
+
+	// get email user information to verify there is no subscription yet
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var unsubscribedEmailUser store.User
+	err = json.Unmarshal(body, &unsubscribedEmailUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "good@example.com test user", ID: "email_f5dfe9d2e6bd75fc74ea5fabf273b45b5baeb195", EmailSubscription: false,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, unsubscribedEmailUser)
+
+	// confirm email via subscribe call, no email notification is expected
+	req, err = http.NewRequest(
+		http.MethodPost,
+		ts.URL+"/api/v1/email/subscribe",
+		io.NopCloser(strings.NewReader(`{"site": "remark42", "address": "good@example.com"}`)),
+	)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	// wait for mock notification Submit to kick off
+	time.Sleep(time.Millisecond * 30)
+	require.Equal(t, 2, len(mockDestination.GetVerify()), "no new verification email was sent")
+
+	// get email user information to verify the subscription happened without the confirmation call
+	req, err = http.NewRequest(
+		http.MethodGet,
+		ts.URL+"/api/v1/user?site=remark42",
+		http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", emailUserToken)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	body, err = io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+	var subscribedEmailUser store.User
+	err = json.Unmarshal(body, &subscribedEmailUser)
+	assert.NoError(t, err)
+	assert.Equal(t, store.User{Name: "good@example.com test user", ID: "email_f5dfe9d2e6bd75fc74ea5fabf273b45b5baeb195", EmailSubscription: true,
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, subscribedEmailUser)
 }
 
 func TestRest_TelegramNotification(t *testing.T) {
@@ -944,7 +1209,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -1023,8 +1288,8 @@ func TestRest_TelegramNotification(t *testing.T) {
 	body, err = io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
-	require.Equal(t, http.StatusInternalServerError, resp.StatusCode, string(body))
-	require.Equal(t, `{"code":0,"details":"can't set telegram for user","error":"not verified"}`+"\n", string(body))
+	require.Equal(t, http.StatusNotFound, resp.StatusCode, string(body))
+	require.Equal(t, `{"code":0,"details":"request is not verified yet","error":"not verified"}`+"\n", string(body))
 
 	mockTlgrm.notVerified = false
 
@@ -1072,7 +1337,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	var user store.User
 	err = json.Unmarshal(body, &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
 
 	// create child comment from another user, telegram notification expected
@@ -1131,7 +1396,7 @@ func TestRest_UserAllData(t *testing.T) {
 	defer teardown()
 
 	// write 3 comments
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
@@ -1156,13 +1421,13 @@ func TestRest_UserAllData(t *testing.T) {
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
 	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 	ungzBody, err := io.ReadAll(ungzReader)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	strUungzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUungzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 3, strings.Count(strUungzBody, `"text":`), "3 comments inside")
 
 	parsed := struct {
@@ -1172,7 +1437,7 @@ func TestRest_UserAllData(t *testing.T) {
 
 	err = json.Unmarshal(ungzBody, &parsed)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, parsed.Info)
 	assert.Equal(t, 3, len(parsed.Comments))
 
@@ -1188,7 +1453,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 
@@ -1215,7 +1480,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	strUngzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUngzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 51, strings.Count(strUngzBody, `"text":`), "51 comments inside")
 }
 
@@ -1239,12 +1504,12 @@ func TestRest_DeleteMe(t *testing.T) {
 	err = json.Unmarshal(body, &m)
 	assert.NoError(t, err)
 	assert.Equal(t, "remark42", m["site"])
-	assert.Equal(t, "dev", m["user_id"])
+	assert.Equal(t, "provider1_dev", m["user_id"])
 
 	tkn := m["token"]
 	claims, err := srv.Authenticator.TokenService().Parse(tkn)
 	assert.NoError(t, err)
-	assert.Equal(t, "dev", claims.User.ID)
+	assert.Equal(t, "provider1_dev", claims.User.ID)
 	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+tkn, m["link"])
 
 	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), http.NoBody)

backend/app/rest/api/rest_public.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/skip2/go-qrcode"
@@ -50,6 +50,8 @@ type pubStore interface {
 
 // GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
+//
+// When `url` parameter is not set (e.g. request is for site-wide comments), does not return deleted comments.
 func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
@@ -77,22 +79,36 @@ func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
 		comments = s.applyView(comments, view)
+
+		var commentsInfo store.PostInfo
+		if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
+			commentsInfo = info
+		}
+
+		if !since.IsZero() { // if since is set, number of comments can be different from total in the DB
+			commentsInfo.Count = 0
+			for _, c := range comments {
+				if !c.Deleted {
+					commentsInfo.Count++
+				}
+			}
+		}
+
+		// post might be readonly without any comments, Info call will fail then and ReadOnly flag should be checked separately
+		if !commentsInfo.ReadOnly && locator.URL != "" && s.dataService.IsReadOnly(locator) {
+			commentsInfo.ReadOnly = true
+		}
+
 		var b []byte
 		switch format {
 		case "tree":
-			tree := service.MakeTree(comments, sort, s.readOnlyAge)
-			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*service.Node{}
+			withInfo := treeWithInfo{Tree: service.MakeTree(comments, sort), Info: commentsInfo}
+			if withInfo.Nodes == nil { // eliminate json nil serialization
+				withInfo.Nodes = []*service.Node{}
 			}
-			if s.dataService.IsReadOnly(locator) {
-				tree.Info.ReadOnly = true
-			}
-			b, e = encodeJSONWithHTML(tree)
+			b, e = encodeJSONWithHTML(withInfo)
 		default:
-			withInfo := commentsWithInfo{Comments: comments}
-			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
-				withInfo.Info = info
-			}
+			withInfo := commentsWithInfo{Comments: comments, Info: commentsInfo}
 			b, e = encodeJSONWithHTML(withInfo)
 		}
 		return b, e

backend/app/rest/api/rest_public_test.go

@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -76,13 +78,31 @@ func TestRest_Preview(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat",
+		`{"code":20,"details":"can't load picture from the comment",`+
+			`"error":"can't get image stats for dev_user/bad_picture: stat`,
 	)
 	assert.Contains(t,
 		string(b),
 		"/pics-remark42/staging/dev_user/62/bad_picture: no such file or directory\"}\n",
 	)
+
+	// test quotes with and without smartypants
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>«quoted» text</p>\n", string(b))
+
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>&#34;quoted&#34; text</p>\n", string(b))
 }
 
 func TestRest_PreviewWithWrongImage(t *testing.T) {
@@ -97,8 +117,8 @@ func TestRest_PreviewWithWrongImage(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat ",
+		`{"code":20,"details":"can't load picture from the comment",`+
+			`"error":"can't get image stats for dev_user/bad_picture: stat `,
 	)
 	assert.Contains(t,
 		string(b),
@@ -131,10 +151,10 @@ BKT
 	assert.NoError(t, err)
 	assert.Equal(t,
 		`<h1>h1</h1>
-<pre class="chroma"><code><span class="line"><span class="cl">func TestRest_Preview(t *testing.T) {
-</span></span><span class="line"><span class="cl">srv, ts := prep(t)
-</span></span><span class="line"><span class="cl">  require.NotNil(t, srv)
-</span></span><span class="line"><span class="cl">}
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="k">func</span> <span class="n">TestRest_Preview</span><span class="p">(</span><span class="n">t</span> <span class="o">*</span><span class="n">testing</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="p">{</span>
+</span></span><span class="line"><span class="cl"><span class="n">srv</span><span class="p">,</span> <span class="n">ts</span> <span class="p">:</span><span class="o">=</span> <span class="n">prep</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+</span></span><span class="line"><span class="cl">  <span class="n">require</span><span class="o">.</span><span class="n">NotNil</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">srv</span><span class="p">)</span>
+</span></span><span class="line"><span class="cl"><span class="p">}</span>
 </span></span></code></pre>`,
 		string(b))
 	assert.NoError(t, resp.Body.Close())
@@ -209,7 +229,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -235,7 +255,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.NoError(t, err)
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
@@ -278,7 +298,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -286,7 +306,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = service.Tree{}
+	tree = treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -325,8 +345,8 @@ func TestRest_FindUserView(t *testing.T) {
 	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
-	assert.Equal(t, "dev", comments.Comments[0].User.ID)
-	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[1].User.ID)
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.Equal(t, "", comments.Comments[1].Text)
 
@@ -440,7 +460,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code, "noting for user blah")
 	assert.Equal(t, `{"comments":[],"count":0}`+"\n", comments)
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -459,7 +479,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev&skip=1&limit=2")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev&skip=1&limit=2")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -477,6 +497,203 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 }
 
+func TestRest_FindUserComments_CWE_918(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"radio-t.com"}) // required for extracting the title, bad URL test
+	defer srv.DataService.TitleExtractor.Close()
+	defer teardown()
+
+	backendRequestedArbitraryServer := false
+	arbitraryServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		t.Logf("request received: %+v", r)
+		backendRequestedArbitraryServer = true
+	}))
+	defer arbitraryServer.Close()
+
+	arbitraryURLComment := store.Comment{Text: "arbitrary URL request test",
+		Locator: store.Locator{SiteID: "remark42", URL: arbitraryServer.URL}}
+
+	assert.False(t, backendRequestedArbitraryServer)
+	addComment(t, arbitraryURLComment, ts)
+	assert.False(t, backendRequestedArbitraryServer,
+		"no request is expected to the test server as it's not in the list of the allowed domains for the title extractor")
+
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
+	assert.Equal(t, http.StatusOK, code)
+
+	resp := struct {
+		Comments []store.Comment
+		Count    int
+	}{}
+
+	err := json.Unmarshal([]byte(res), &resp)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(resp.Comments), "should have 2 comments")
+
+	assert.Equal(t, "", resp.Comments[0].PostTitle, "empty from the first post")
+	assert.Equal(t, arbitraryServer.URL, resp.Comments[0].Locator.URL, "arbitrary URL provided by the request")
+}
+
+func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
+	// test that comment counting is consistent between tree and plain formats
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	commentLocator := store.Locator{URL: "test-url", SiteID: "remark42"}
+
+	// vote for comment multiple times
+	setScore := func(locator store.Locator, id string, val int) {
+		abs := func(x int) int {
+			if x < 0 {
+				return -x
+			}
+			return x
+		}
+		for i := 0; i < abs(val); i++ {
+			_, err := srv.DataService.Vote(service.VoteReq{
+				Locator:   locator,
+				CommentID: id,
+				// unique user ID is needed for correct counting of controversial votes
+				UserID: "user" + strconv.Itoa(val) + strconv.Itoa(i),
+				Val:    val > 0,
+			})
+			require.NoError(t, err)
+		}
+	}
+
+	// Adding initial comments (8 to test-url and 1 to another-url) and voting, and delete two of comments to the first post.
+	// With sleep so that at least few millisecond pass between each comment
+	// and later we would be able to use that in "since" filter with millisecond precision
+	ids := make([]string, 9)
+	timestamps := make([]time.Time, 9)
+	c1 := store.Comment{Text: "top-level comment 1", Locator: commentLocator}
+	ids[0], timestamps[0] = addCommentGetCreatedTime(t, c1, ts)
+	// #3 by score
+	setScore(commentLocator, ids[0], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c2 := store.Comment{Text: "top-level comment 2", Locator: commentLocator}
+	ids[1], timestamps[1] = addCommentGetCreatedTime(t, c2, ts)
+	// #2 by score
+	setScore(commentLocator, ids[1], 2)
+	time.Sleep(time.Millisecond * 5)
+
+	c3 := store.Comment{Text: "second-level comment 1", ParentID: ids[0], Locator: commentLocator}
+	ids[2], timestamps[2] = addCommentGetCreatedTime(t, c3, ts)
+	// #1 by score
+	setScore(commentLocator, ids[2], 10)
+	time.Sleep(time.Millisecond * 5)
+
+	c4 := store.Comment{Text: "third-level comment 1", ParentID: ids[2], Locator: commentLocator}
+	ids[3], timestamps[3] = addCommentGetCreatedTime(t, c4, ts)
+	// #5 by score, #1 by controversy
+	setScore(commentLocator, ids[3], 4)
+	setScore(commentLocator, ids[3], -4)
+	time.Sleep(time.Millisecond * 5)
+
+	c5 := store.Comment{Text: "second-level comment 2", ParentID: ids[1], Locator: commentLocator}
+	ids[4], timestamps[4] = addCommentGetCreatedTime(t, c5, ts)
+	// #5 by score, #2 by controversy
+	setScore(commentLocator, ids[4], 2)
+	setScore(commentLocator, ids[4], -3)
+	time.Sleep(time.Millisecond * 5)
+
+	c6 := store.Comment{Text: "third-level comment 2", ParentID: ids[4], Locator: commentLocator}
+	ids[5], timestamps[5] = addCommentGetCreatedTime(t, c6, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[5], 10)
+	setScore(commentLocator, ids[5], -10)
+	time.Sleep(time.Millisecond * 5)
+
+	c7 := store.Comment{Text: "top-level comment 3", Locator: commentLocator}
+	ids[6], timestamps[6] = addCommentGetCreatedTime(t, c7, ts)
+	// #6 by score, #4 by controversy
+	setScore(commentLocator, ids[6], -3)
+	setScore(commentLocator, ids[6], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c8 := store.Comment{Text: "second-level comment 3", ParentID: ids[6], Locator: commentLocator}
+	ids[7], timestamps[7] = addCommentGetCreatedTime(t, c8, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[7], -20)
+
+	c9 := store.Comment{Text: "comment to post 2", Locator: store.Locator{URL: "another-url", SiteID: "remark42"}}
+	ids[8], timestamps[8] = addCommentGetCreatedTime(t, c9, ts)
+	// #7 by score
+	setScore(store.Locator{URL: "another-url", SiteID: "remark42"}, ids[8], -25)
+
+	// delete two comments bringing the total from 9 to 6
+	err := srv.DataService.Delete(commentLocator, ids[7], store.SoftDelete)
+	assert.NoError(t, err)
+	err = srv.DataService.Delete(commentLocator, ids[5], store.HardDelete)
+	assert.NoError(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
+
+	commentLocator.URL = "readonly-test"
+	// set post without comments to read-only
+	assert.NoError(t, srv.DataService.SetReadOnly(commentLocator, true))
+
+	sinceTenSecondsAgo := strconv.FormatInt(time.Now().Add(-time.Second*10).UnixNano()/1000000, 10)
+	sinceTS := make([]string, 9)
+	formattedTS := make([]string, 9)
+	for i, created := range timestamps {
+		sinceTS[i] = strconv.FormatInt(created.UnixNano()/1000000, 10)
+		formattedTS[i] = created.Format(time.RFC3339Nano)
+	}
+	t.Logf("last timestamp: %v", timestamps[7])
+
+	testCases := []struct {
+		params       string
+		expectedBody string
+	}{
+		{"", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=plain", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"format=plain&url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[0], fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[0], fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[1], fmt.Sprintf(`"info":{"count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[1], fmt.Sprintf(`"info":{"url":"test-url","count":5,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[4], fmt.Sprintf(`"info":{"count":3,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[4], fmt.Sprintf(`"info":{"url":"test-url","count":2,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=tree", `"info":{"count":7`},
+		{"format=tree&url=test-url", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=+time", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=+time", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=-score", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=-score", `"info":{"url":"test-url","count":6`},
+		{"sort=+time", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-time", fmt.Sprintf(`"score":1,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[0])},
+		{"sort=+score", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[2])},
+		{"sort=+score&url=test-url", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[2])},
+		{"sort=-score", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-score&url=test-url", fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[6])},
+		{"sort=-time&since=" + sinceTS[4], fmt.Sprintf(`"score":-1,"vote":0,"controversy":2.924017738212866,"time":%q}],"info":{"count":3`, formattedTS[4])},
+		{"sort=-score&since=" + sinceTS[3], fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":4`, formattedTS[8])},
+		{"sort=-score&url=test-url&since=" + sinceTS[3], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":3`, formattedTS[6])},
+		{"sort=+controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":1`, formattedTS[6])},
+		// three comments of which last one deleted and doesn't have controversy so returned last
+		{"sort=-controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":0,"vote":0,"time":%q,"delete":true}],"info":{"url":"test-url","count":1`, formattedTS[7])},
+		// test readonly status for the post without comments
+		{"url=readonly-test", `"info":{"count":0,"read_only":true`},
+		{"format=tree&url=readonly-test", `"info":{"count":0,"read_only":true`},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.params, func(t *testing.T) {
+			url := fmt.Sprintf(ts.URL+"/api/v1/find?site=remark42&%s", tc.params)
+			body, code := get(t, url)
+			assert.Equal(t, http.StatusOK, code)
+			assert.Contains(t, body, tc.expectedBody)
+			t.Log(body)
+			// prevent hit limiter from engaging
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -486,7 +703,7 @@ func TestRest_UserInfo(t *testing.T) {
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", Picture: "http://example.com/pic.png",
 		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 

backend/app/rest/api/rest_test.go

@@ -18,8 +18,9 @@ import (
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -37,11 +38,18 @@ import (
 	"github.com/umputun/remark42/backend/app/store/service"
 )
 
-var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
+// To generate a token, enter one of the tokens here into https://jwt.io, change the secret to one you're using in your test
+// ("secret" in case of startupT), and alter the fields you want to be changed.
+
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.dirTS_ahSF6375sdO2iodm2K2UmRTzQNQMFiHuTQCVs`
+
+var dev2Token = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYyIiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.qsR_PupfjIq7uw0eAuyGV8nsUoMx9v541c9olnRInRQ`
 
 var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
 
-var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
+var emailUserToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6Imdvb2RAZXhhbXBsZS5jb20gdGVzdCB1c2VyIiwiaWQiOiJlbWFpbF9mNWRmZTlkMmU2YmQ3NWZjNzRlYTVmYWJmMjczYjQ1YjViYWViMTk1IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Imdvb2RAZXhhbXBsZS5jb20ifX0.vH2HN1JpuXL8okTJq1A-zGHQ-l2ILcwxvDDEmu2zwks`
+
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJwcm92aWRlcjFfZGV2IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.X-lvnHvBz6VfEbVV4f-bjcZuLY5pYtvEansk_TQMrX8`
 
 var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
 
@@ -120,7 +128,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -170,7 +178,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}
@@ -186,7 +194,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 }
 
 func TestRest_rejectAnonUser(t *testing.T) {
-	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		fmt.Fprintln(w, "Hello")
 	}))))
 	defer ts.Close()
@@ -296,10 +304,10 @@ func TestRest_cacheControl(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", tt.url, nil)
+			req := httptest.NewRequest("GET", tt.url, http.NoBody)
 			w := httptest.NewRecorder()
 
-			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -312,30 +320,54 @@ func TestRest_cacheControl(t *testing.T) {
 }
 
 func TestRest_frameAncestors(t *testing.T) {
-	tbl := []struct {
-		hosts  []string
-		header string
-	}{
-		{[]string{"http://example.com"}, "frame-ancestors http://example.com;"},
-		{[]string{}, ""},
-		{[]string{"http://example.com", "http://example2.com"}, "frame-ancestors http://example.com http://example2.com;"},
-	}
+	ts, _, teardown := startupT(t, func(o *Rest) {
+		o.AllowedAncestors = []string{"'self'", "https://example.com"}
+	})
 
-	for i, tt := range tbl {
-		tt := tt
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
-			w := httptest.NewRecorder()
+	// Test case with frame-ancestors
+	client := http.Client{}
+	resp, err := client.Get(ts.URL + "/web/index.html")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "frame-ancestors 'self' https://example.com;")
+	teardown()
 
-			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-			h.ServeHTTP(w, req)
-			resp := w.Result()
-			assert.Equal(t, http.StatusOK, resp.StatusCode)
-			assert.NoError(t, resp.Body.Close())
-			t.Logf("%+v", resp.Header)
-			assert.Equal(t, tt.header, resp.Header.Get("Content-Security-Policy"))
-		})
-	}
+	// Test case without frame-ancestors
+	ts, _, teardown = startupT(t, func(srv *Rest) {
+		srv.AllowedAncestors = []string{}
+	})
+	defer teardown()
+	resp, err = client.Get(ts.URL + "/web/index.html")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "frame-ancestors *;")
+}
+
+// check CSP, img-src should be 'self' with proxy enabled and * without it
+func TestRest_securityHeaders(t *testing.T) {
+	ts, _, teardown := startupT(t)
+
+	// with proxy disabled
+	client := http.Client{}
+	resp, err := client.Get(ts.URL + "/web/index.html")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "img-src *;")
+	teardown()
+
+	// check CSP with proxy enabled
+	ts, _, teardown = startupT(t, func(srv *Rest) {
+		srv.ExternalImageProxy = true
+	})
+	defer teardown()
+	resp, err = client.Get(ts.URL + "/web/index.html")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "img-src 'self';")
 }
 
 func TestRest_subscribersOnly(t *testing.T) {
@@ -358,12 +390,12 @@ func TestRest_subscribersOnly(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req := httptest.NewRequest("GET", "http://example.com", http.NoBody)
 			if tt.setUser {
 				req = token.SetUserInfo(req, tt.user)
 			}
 			w := httptest.NewRecorder()
-			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, tt.status, resp.StatusCode)
@@ -395,7 +427,7 @@ func Test_validEmailAuth(t *testing.T) {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			req := httptest.NewRequest("GET", "http://example.com"+tt.req, http.NoBody)
 			w := httptest.NewRecorder()
-			h := validEmailAuth()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := validEmailAuth()(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, tt.status, resp.StatusCode)
@@ -430,7 +462,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "remark42"})
 	require.NoError(t, err)
 
-	memCache := cache.NewScache(cache.NewNopCache())
+	memCache := cache.NewScache[[]byte](cache.NewNopCache[[]byte]())
 
 	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
@@ -450,7 +482,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
+			SecretReader: token.SecretFunc(func(string) (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
@@ -479,16 +511,26 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 			Cache:             memCache,
 			KeyStore:          astore,
 		},
-		NotifyService: notify.NopService,
-		EmojiEnabled:  true,
+		NotifyService:    notify.NopService,
+		EmojiEnabled:     true,
+		openRouteLimiter: 100,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
+	// add some providers. Needed because we don't allow users with unlisted providers to authenticate
+	providers := []string{"provider1", "anonymous", "github", "email"}
+	for _, p := range providers {
+		srv.Authenticator.AddDirectProvider(p, provider.CredCheckerFunc(func(_, _ string) (ok bool, err error) {
+			return true, nil
+		}))
+	}
+
 	for _, h := range srvHook {
 		h(srv)
 	}
 
-	ts = httptest.NewServer(srv.routes())
+	routes := srv.routes()
+	ts = httptest.NewServer(routes)
 
 	teardown = func() {
 		ts.Close()
@@ -547,6 +589,20 @@ func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	return string(b), r.StatusCode
 }
 
+func getWithDev2Auth(t *testing.T, url string) (body string, code int) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("GET", url, http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", dev2Token)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	b, err := io.ReadAll(r.Body)
+	assert.NoError(t, err)
+	require.NoError(t, r.Body.Close())
+	return string(b), r.StatusCode
+}
+
 func getWithAdminAuth(t *testing.T, url string) (response string, statusCode int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	defer client.CloseIdleConnections()
@@ -569,7 +625,7 @@ func post(t *testing.T, url, body string) (*http.Response, error) {
 	return client.Do(req)
 }
 
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+func addCommentGetCreatedTime(t *testing.T, c store.Comment, ts *httptest.Server) (id string, created time.Time) {
 	b, err := json.Marshal(c)
 	require.NoError(t, err, "can't marshal comment %+v", c)
 
@@ -589,7 +645,14 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	err = json.Unmarshal(b, &crResp)
 	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
+	created, err = time.Parse(time.RFC3339, crResp["time"].(string))
+	require.NoError(t, err)
+	return crResp["id"].(string), created
+}
+
+func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+	id, _ := addCommentGetCreatedTime(t, c, ts)
+	return id
 }
 
 func requireAdminOnly(t *testing.T, req *http.Request) {
@@ -628,5 +691,9 @@ func waitForHTTPSServerStart(port int) {
 }
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(
+		m,
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/rest/api/rss.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"github.com/gorilla/feeds"
 
@@ -56,7 +56,6 @@ func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-
 	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}

backend/app/rest/api/ssl_test.go

@@ -21,7 +21,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -56,7 +56,7 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}

backend/app/rest/api/testdata/commento.json

@@ -0,0 +1,116 @@
+{
+  "version": 1,
+  "comments": [
+    {
+      "commentHex": "e7a2ef4b4aa1414a7ee65a989889aaecd9d5e7e3bca598ea7a967b4dbcaa8e11",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2022-10-25T07:25:46.807555Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "a29e741145daceb4ca5b3e5e279e05b56f73c04703d93b944718ef757e15317f",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-07-26T12:24:55.058552Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-31T11:03:25.403282Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "6d3bb64ff73b5f9d6a959212ffde472a51abf8bdefaa5ed843659796bceef9de",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:23:47.112062Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-23T12:33:03.370182Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "d0ad6f11cf0c5f8e17457a378a6bb789f412c6b7ef7ada4ae06ec8451f7a18aa",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-18T01:18:38.193625Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "098960fd01c1fc7c0d3ea428f52fab97ea5c18aa52f3565bba679224daddc687",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:24:04.639965Z",
+      "direction": 0,
+      "deleted": false
+    }
+  ],
+  "commenters": [
+    {
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "email": "undefined",
+      "name": "blank",
+      "link": "undefined",
+      "photo": "undefined",
+      "provider": "anon",
+      "joinDate": "2022-06-09T15:54:29.865919Z",
+      "isModerator": false,
+      "deleted": false
+    }
+  ]
+}

backend/app/rest/proxy/image.go

@@ -57,7 +57,7 @@ func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]stri
 		return nil, fmt.Errorf("can't create document: %w", err)
 	}
 	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		if im, ok := s.Attr("src"); ok {
 			if imgSrcPred(im) {
 				result = append(result, im)
@@ -98,6 +98,10 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 	if img == nil {
 		img, err = p.downloadImage(context.Background(), imgURL)
 		if err != nil {
+			if strings.Contains(err.Error(), "invalid content type") {
+				rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid content type", rest.ErrImgNotFound)
+				return
+			}
 			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
 			return
 		}
@@ -159,12 +163,17 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error)
 	if err != nil {
 		return nil, fmt.Errorf("can't download image %s: %w", imgURL, err)
 	}
-	defer resp.Body.Close()
+	defer resp.Body.Close() //nolint gosec // we don't care about response body
 
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
 	}
 
+	contentType := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(contentType, "image/") {
+		return nil, fmt.Errorf("invalid content type %s", contentType)
+	}
+
 	imgData, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read image body")

backend/app/rest/proxy/image_test.go

@@ -95,7 +95,7 @@ func TestImage_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	// no image supposed to be cached
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",
@@ -108,31 +108,45 @@ func TestImage_Routes(t *testing.T) {
 	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
-	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+	t.Run("valid image", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+		assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	})
 
-	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	t.Run("no image", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+	})
 
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+	t.Run("bad encoding", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 2, len(imageStore.LoadCalls()))
+	})
 
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.NoError(t, err)
-	assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	assert.Equal(t, 2, len(imageStore.LoadCalls()))
+	t.Run("non-image reference", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte("https://google.com"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 3, len(imageStore.LoadCalls()))
+	})
 }
 
 func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		RemarkURL:    "https://demo.remark42.com",
 		RoutePath:    "/api/v1/proxy",
@@ -158,10 +172,10 @@ func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
 
 func TestImage_RoutesCachingImage(t *testing.T) {
 	imageStore := image.StoreMock{
-		LoadFunc: func(id string) ([]byte, error) {
+		LoadFunc: func(string) ([]byte, error) {
 			return nil, nil
 		},
-		SaveFunc: func(id string, img []byte) error {
+		SaveFunc: func(string, []byte) error {
 			return nil
 		},
 	}
@@ -196,7 +210,7 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 func TestImage_RoutesUsingCachedImage(t *testing.T) {
 	// In order to validate that cached data used cache "will return" some other data from what http server would
 	testImage := []byte(fmt.Sprintf("%256s", "X"))
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) {
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) {
 		return testImage, nil
 	}}
 	img := Image{
@@ -226,7 +240,7 @@ func TestImage_RoutesUsingCachedImage(t *testing.T) {
 
 func TestImage_RoutesTimedOut(t *testing.T) {
 	// no image supposed to be cached
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",

backend/app/store/admin/admin.go

@@ -8,6 +8,9 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
+//go:generate moq --out admin_mock.go . Store
+
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key(siteID string) (key string, err error)

backend/app/store/admin/admin_mock.go

@@ -0,0 +1,256 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package admin
+
+import (
+	"sync"
+)
+
+// Ensure, that StoreMock does implement Store.
+// If this is not the case, regenerate this file with moq.
+var _ Store = &StoreMock{}
+
+// StoreMock is a mock implementation of Store.
+//
+//	func TestSomethingThatUsesStore(t *testing.T) {
+//
+//		// make and configure a mocked Store
+//		mockedStore := &StoreMock{
+//			AdminsFunc: func(siteID string) ([]string, error) {
+//				panic("mock out the Admins method")
+//			},
+//			EmailFunc: func(siteID string) (string, error) {
+//				panic("mock out the Email method")
+//			},
+//			EnabledFunc: func(siteID string) (bool, error) {
+//				panic("mock out the Enabled method")
+//			},
+//			KeyFunc: func(siteID string) (string, error) {
+//				panic("mock out the Key method")
+//			},
+//			OnEventFunc: func(siteID string, et EventType) error {
+//				panic("mock out the OnEvent method")
+//			},
+//		}
+//
+//		// use mockedStore in code that requires Store
+//		// and then make assertions.
+//
+//	}
+type StoreMock struct {
+	// AdminsFunc mocks the Admins method.
+	AdminsFunc func(siteID string) ([]string, error)
+
+	// EmailFunc mocks the Email method.
+	EmailFunc func(siteID string) (string, error)
+
+	// EnabledFunc mocks the Enabled method.
+	EnabledFunc func(siteID string) (bool, error)
+
+	// KeyFunc mocks the Key method.
+	KeyFunc func(siteID string) (string, error)
+
+	// OnEventFunc mocks the OnEvent method.
+	OnEventFunc func(siteID string, et EventType) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Admins holds details about calls to the Admins method.
+		Admins []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Email holds details about calls to the Email method.
+		Email []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Enabled holds details about calls to the Enabled method.
+		Enabled []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Key holds details about calls to the Key method.
+		Key []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// OnEvent holds details about calls to the OnEvent method.
+		OnEvent []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+			// Et is the et argument value.
+			Et EventType
+		}
+	}
+	lockAdmins  sync.RWMutex
+	lockEmail   sync.RWMutex
+	lockEnabled sync.RWMutex
+	lockKey     sync.RWMutex
+	lockOnEvent sync.RWMutex
+}
+
+// Admins calls AdminsFunc.
+func (mock *StoreMock) Admins(siteID string) ([]string, error) {
+	if mock.AdminsFunc == nil {
+		panic("StoreMock.AdminsFunc: method is nil but Store.Admins was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockAdmins.Lock()
+	mock.calls.Admins = append(mock.calls.Admins, callInfo)
+	mock.lockAdmins.Unlock()
+	return mock.AdminsFunc(siteID)
+}
+
+// AdminsCalls gets all the calls that were made to Admins.
+// Check the length with:
+//
+//	len(mockedStore.AdminsCalls())
+func (mock *StoreMock) AdminsCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockAdmins.RLock()
+	calls = mock.calls.Admins
+	mock.lockAdmins.RUnlock()
+	return calls
+}
+
+// Email calls EmailFunc.
+func (mock *StoreMock) Email(siteID string) (string, error) {
+	if mock.EmailFunc == nil {
+		panic("StoreMock.EmailFunc: method is nil but Store.Email was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEmail.Lock()
+	mock.calls.Email = append(mock.calls.Email, callInfo)
+	mock.lockEmail.Unlock()
+	return mock.EmailFunc(siteID)
+}
+
+// EmailCalls gets all the calls that were made to Email.
+// Check the length with:
+//
+//	len(mockedStore.EmailCalls())
+func (mock *StoreMock) EmailCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEmail.RLock()
+	calls = mock.calls.Email
+	mock.lockEmail.RUnlock()
+	return calls
+}
+
+// Enabled calls EnabledFunc.
+func (mock *StoreMock) Enabled(siteID string) (bool, error) {
+	if mock.EnabledFunc == nil {
+		panic("StoreMock.EnabledFunc: method is nil but Store.Enabled was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEnabled.Lock()
+	mock.calls.Enabled = append(mock.calls.Enabled, callInfo)
+	mock.lockEnabled.Unlock()
+	return mock.EnabledFunc(siteID)
+}
+
+// EnabledCalls gets all the calls that were made to Enabled.
+// Check the length with:
+//
+//	len(mockedStore.EnabledCalls())
+func (mock *StoreMock) EnabledCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEnabled.RLock()
+	calls = mock.calls.Enabled
+	mock.lockEnabled.RUnlock()
+	return calls
+}
+
+// Key calls KeyFunc.
+func (mock *StoreMock) Key(siteID string) (string, error) {
+	if mock.KeyFunc == nil {
+		panic("StoreMock.KeyFunc: method is nil but Store.Key was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockKey.Lock()
+	mock.calls.Key = append(mock.calls.Key, callInfo)
+	mock.lockKey.Unlock()
+	return mock.KeyFunc(siteID)
+}
+
+// KeyCalls gets all the calls that were made to Key.
+// Check the length with:
+//
+//	len(mockedStore.KeyCalls())
+func (mock *StoreMock) KeyCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockKey.RLock()
+	calls = mock.calls.Key
+	mock.lockKey.RUnlock()
+	return calls
+}
+
+// OnEvent calls OnEventFunc.
+func (mock *StoreMock) OnEvent(siteID string, et EventType) error {
+	if mock.OnEventFunc == nil {
+		panic("StoreMock.OnEventFunc: method is nil but Store.OnEvent was just called")
+	}
+	callInfo := struct {
+		SiteID string
+		Et     EventType
+	}{
+		SiteID: siteID,
+		Et:     et,
+	}
+	mock.lockOnEvent.Lock()
+	mock.calls.OnEvent = append(mock.calls.OnEvent, callInfo)
+	mock.lockOnEvent.Unlock()
+	return mock.OnEventFunc(siteID, et)
+}
+
+// OnEventCalls gets all the calls that were made to OnEvent.
+// Check the length with:
+//
+//	len(mockedStore.OnEventCalls())
+func (mock *StoreMock) OnEventCalls() []struct {
+	SiteID string
+	Et     EventType
+} {
+	var calls []struct {
+		SiteID string
+		Et     EventType
+	}
+	mock.lockOnEvent.RLock()
+	calls = mock.calls.OnEvent
+	mock.lockOnEvent.RUnlock()
+	return calls
+}

backend/app/store/admin/admin_test.go

@@ -6,6 +6,20 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStaticStore_StoreWithoutSites(t *testing.T) {
+	var ks Store = NewStaticKeyStore("key123")
+	enabled, err := ks.Enabled("any")
+	assert.NoError(t, err)
+	assert.True(t, enabled, "on empty store all sites are enabled")
+	assert.NoError(t, ks.OnEvent("test", EvCreate), "static store does nothing OnEvent")
+
+	// empty key
+	ks = NewStaticKeyStore("")
+	key, err := ks.Key("any")
+	assert.Error(t, err, "empty key")
+	assert.Empty(t, key)
+}
+
 func TestStaticStore_Get(t *testing.T) {
 	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"},
 		[]string{"123", "xyz"}, "aa@example.com")

backend/app/store/comment.go

@@ -45,9 +45,9 @@ type Edit struct {
 
 // PostInfo holds summary for given post url
 type PostInfo struct {
-	URL      string    `json:"url"`
+	URL      string    `json:"url,omitempty"` // can be attached to site-wide comments but won't be set then
 	Count    int       `json:"count"`
-	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"`
+	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"` // can be attached to site-wide comments but won't be set then
 	FirstTS  time.Time `json:"first_time,omitempty" bson:"first_time,omitempty"`
 	LastTS   time.Time `json:"last_time,omitempty" bson:"last_time,omitempty"`
 }
@@ -98,6 +98,7 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 	c.Text = ""
 	c.Orig = ""
 	c.Score = 0
+	c.Controversy = 0
 	c.Votes = map[string]bool{}
 	c.VotedIPs = make(map[string]VotedIPInfo)
 	c.Edit = nil
@@ -118,9 +119,11 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 func (c *Comment) Sanitize() {
 	p := bluemonday.UGCPolicy()
 	p.AllowAttrs("class").Matching(regexp.MustCompile("^chroma$")).OnElements("pre")
+	// special case for embedding the quotes from Twitter
+	p.AllowAttrs("class").Matching(regexp.MustCompile("^twitter-tweet$")).OnElements("blockquote")
 	// this is list of <span> tag classes which could be produced by chroma code renderer
-	// source: https://github.com/alecthomas/chroma/blob/cc2dd5b/types.go#L211-L307
-	const codeSpanClassRegex = "^(bg|chroma|line|ln|lnt|hl|lntable|lntd|cl|w|err|x|k|kc" +
+	// source: https://github.com/alecthomas/chroma/blob/c263f6f/types.go#L209-L306
+	const codeSpanClassRegex = "^(bg|chroma|line|ln|lnt|hl|lntable|lntd|lnlinks|cl|w|err|x|k|kc" +
 		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
 		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
 		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
@@ -144,7 +147,7 @@ func (c *Comment) Snippet(limit int) string {
 	if size < limit {
 		return cleanText
 	}
-	snippet := []rune(cleanText)[:size]
+	snippet := []rune(cleanText)[:limit]
 	// go back in snippet and found the first space
 	for i := len(snippet) - 1; i >= 0; i-- {
 		if snippet[i] == ' ' {
@@ -152,6 +155,10 @@ func (c *Comment) Snippet(limit int) string {
 			break
 		}
 	}
+	// Don't add a space if comment is just a one single word which has been truncated.
+	if len(snippet) == limit {
+		return string(snippet) + "..."
+	}
 	return string(snippet) + " ..."
 }
 
@@ -176,8 +183,8 @@ func (c *Comment) escapeHTMLWithSome(inp string) string {
 	return res
 }
 
-// SanitizeText used to sanitize any input string
+// SanitizeText used to sanitize any input string, and removes any HTML tags
 func (c *Comment) SanitizeText(inp string) string {
-	clean := bluemonday.UGCPolicy().Sanitize(inp)
-	return c.escapeHTMLWithSome(clean)
+	clean := bluemonday.StrictPolicy().Sanitize(inp)
+	return strings.TrimSpace(c.escapeHTMLWithSome(clean))
 }

backend/app/store/comment_test.go

@@ -22,7 +22,7 @@ func TestComment_Sanitize(t *testing.T) {
 			},
 			out: Comment{
 				Text: "blah XSS\n\t",
-				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
+				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name"},
 			},
 		},
 		{
@@ -88,6 +88,18 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah blah", PostTitle: "<script>alert()</script>something"},
 			out: Comment{Text: "blah blah", PostTitle: "something"},
 		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "<a href=\"https://example.com\">test</a>"},
+			out: Comment{Text: "blah blah", PostTitle: "test"},
+		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"}, // link is left as-is, but not rendered as <a>
+			out: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"},
+		},
+		{
+			inp: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw">@marcsheep</a> <a href="https://t.co/ULnVpG8w55">pic.twitter.com/ULnVpG8w55</a></p>&mdash; Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw">November 9, 2020</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>`, PostTitle: "Twitter quote"},
+			out: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn" rel="nofollow">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw" rel="nofollow">@marcsheep</a> <a href="https://t.co/ULnVpG8w55" rel="nofollow">pic.twitter.com/ULnVpG8w55</a></p>— Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw" rel="nofollow">November 9, 2020</a></blockquote> `, PostTitle: "Twitter quote"},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -192,8 +204,8 @@ func TestComment_Snippet(t *testing.T) {
 		{0, "", ""},
 		{-1, "test\nblah", "test blah"},
 		{5, "test\nblah", "test ..."},
-		{5, "xyz12345 xxx", "xyz12345 ..."},
-		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+		{5, "xyz12345 xxx", "xyz12..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 ..."},
 	}
 
 	for i, tt := range tbl {
@@ -259,7 +271,7 @@ func TestComment_sanitizeText(t *testing.T) {
 		},
 		{
 			"<a href=javascript:alert(document.domain)//>xxx</a>",
-			"&lt;a/&gt;xxx&lt;/a&gt;",
+			"xxx",
 		},
 	}
 

backend/app/store/engine/bolt.go

@@ -170,7 +170,7 @@ func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
 				return e
 			}
 
-			return bucket.ForEach(func(k, v []byte) error {
+			return bucket.ForEach(func(_, v []byte) error {
 				comment := store.Comment{}
 				if e = json.Unmarshal(v, &comment); e != nil {
 					return fmt.Errorf("failed to unmarshal: %w", e)
@@ -306,9 +306,8 @@ func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
 		})
 
 		// set read-only from age and manual bucket
-		readOnlyAge := req.ReadOnlyAge
-		info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-		if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly && b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
 			info.ReadOnly = true
 		}
 		return []store.PostInfo{info}, err
@@ -425,11 +424,11 @@ func (b *BoltDB) Close() error {
 }
 
 // Last returns up to max last comments for given siteID
-func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+func (b *BoltDB) lastComments(siteID string, maximum int, since time.Time) (comments []store.Comment, err error) {
 	comments = []store.Comment{}
 
-	if max > lastLimit || max == 0 {
-		max = lastLimit
+	if maximum > lastLimit || maximum == 0 {
+		maximum = lastLimit
 	}
 
 	bdb, err := b.db(siteID)
@@ -467,7 +466,7 @@ func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments
 				continue
 			}
 			comments = append(comments, comment)
-			if len(comments) >= max {
+			if len(comments) >= maximum {
 				break
 			}
 		}
@@ -725,7 +724,7 @@ func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err e
 	err = bdb.View(func(tx *bolt.Tx) error {
 		var entry UserDetailEntry
 		bucket := tx.Bucket([]byte(userDetailsBucketName))
-		return bucket.ForEach(func(userID, value []byte) error {
+		return bucket.ForEach(func(_, value []byte) error {
 			if err = json.Unmarshal(value, &entry); err != nil {
 				return fmt.Errorf("failed to unmarshal entry: %w", e)
 			}
@@ -870,7 +869,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID, userID string, mode store.Dele
 		err = bdb.View(func(tx *bolt.Tx) error {
 			postsBkt := tx.Bucket([]byte(postsBucketName))
 			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+			err = postBkt.ForEach(func(_ []byte, commentVal []byte) error {
 				comment := store.Comment{}
 				if err = json.Unmarshal(commentVal, &comment); err != nil {
 					return fmt.Errorf("failed to unmarshal: %w", err)

backend/app/store/engine/bolt_test.go

@@ -911,6 +911,13 @@ func TestBoltDB_NewFailed(t *testing.T) {
 	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
 }
 
+func TestBoltDB_DoubleClose(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+	assert.NoError(t, b.Close())
+	assert.NoError(t, b.Close(), "second call should not result in panic or errors")
+}
+
 // makes new boltdb, put two records
 func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	_ = os.Remove(testDB)

backend/app/store/engine/remote_test.go

@@ -186,11 +186,17 @@ func TestRemote_Delete(t *testing.T) {
 }
 
 func TestRemote_Close(t *testing.T) {
-	ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Contains(t, string(body), "{\"method\":\"store.close\",\"id\":")
+		t.Logf("req: %s", string(body))
+		_, _ = fmt.Fprint(w, "{}")
+	}))
 	defer ts.Close()
 	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
-	err := c.Close()
-	assert.NoError(t, err)
+	assert.NoError(t, c.Close())
+	assert.NoError(t, c.Close(), "second call should not result in panic or errors")
 }
 
 func testServer(t *testing.T, req, resp string) *httptest.Server {

backend/app/store/formatter.go

@@ -35,24 +35,17 @@ func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter {
 }
 
 // Format comment fields
-func (f *CommentFormatter) Format(c Comment) Comment {
-	c.Text = f.FormatText(c.Text)
+func (f *CommentFormatter) Format(c Comment, raw bool) Comment {
+	c.Text = f.FormatText(c.Text, raw)
 	return c
 }
 
 // FormatText converts text with markdown processor, applies external converters and shortens links
-func (f *CommentFormatter) FormatText(txt string) (res string) {
-	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
-		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
-		bf.BackslashLineBreak | bf.Autolink
-
-	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{
-		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
-	})
-
-	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
-
-	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(extRend)))
+//
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
+func (f *CommentFormatter) FormatText(txt string, raw bool) (res string) {
+	mdExt, rend := GetMdExtensionsAndRenderer(raw)
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
 	res = f.unEscape(res)
 
 	for _, conv := range f.converters {
@@ -64,14 +57,14 @@ func (f *CommentFormatter) FormatText(txt string) (res string) {
 }
 
 // Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
-func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
+func (f *CommentFormatter) shortenAutoLinks(commentHTML string, maximum int) (resHTML string) {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("a").Each(func(i int, s *goquery.Selection) {
+	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
 		if href, ok := s.Attr("href"); ok {
-			if href != s.Text() || len(href) < max+3 || max < 3 {
+			if href != s.Text() || len(href) < maximum+3 || maximum < 3 {
 				return
 			}
 			commentURL, e := url.Parse(href)
@@ -84,7 +77,7 @@ func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTM
 				return
 			}
 
-			short := string([]rune(href)[:max-3])
+			short := string([]rune(href)[:maximum-3])
 			if len(short) < len(host) {
 				short = host
 			}
@@ -117,7 +110,7 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		s.SetAttr("loading", "lazy")
 	})
 	resHTML, err = doc.Find("body").Html()
@@ -126,3 +119,23 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	}
 	return resHTML
 }
+
+// GetMdExtensionsAndRenderer returns blackfriday extensions and renderer used for rendering markdown
+// within store module.
+//
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
+func GetMdExtensionsAndRenderer(raw bool) (bf.Extensions, *bfchroma.Renderer) {
+	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
+		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
+		bf.BackslashLineBreak | bf.Autolink
+
+	flags := bf.HTMLFlags(0)
+	if !raw {
+		flags = bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes
+	}
+
+	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{Flags: flags})
+
+	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
+	return mdExt, extRend
+}

backend/app/store/formatter_test.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -35,30 +36,38 @@ func TestFormatter_FormatText(t *testing.T) {
 			"lazy image",
 		},
 		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
+		{`no_smartpants "quoted" text`, "<p>no_smartpants &#34;quoted&#34; text</p>\n!converted", "normal quotes without smartpants"},
+		{`"quoted" text`, "<p>«quoted» text</p>\n!converted", "normal quotes with smartpants"},
+		{`no_smartpants “quoted” text`, "<p>no_smartpants “quoted” text</p>\n!converted", "curly quotes without smartpants"},
+		{`“quoted” text`, "<p>“quoted” text</p>\n!converted", "curly quotes with smartpants"},
+		{`no_smartpants «quoted» text`, "<p>no_smartpants «quoted» text</p>\n!converted", "French guillemets without smartpants"},
+		{`«quoted» text`, "<p>«quoted» text</p>\n!converted", "French guillemets with smartpants"},
 		{"smth\n```go\nfunc main(aa string) int {return 0}\n```", `<p>smth</p>
-<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
 </span></span></code></pre>!converted`, "code with language"},
-		{"```\ntest_code\n```", `<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl">test_code
+		{"```\ntest_code\n```", `<pre class="chroma"><code><span class="line"><span class="cl">test_code
 </span></span></code></pre>!converted`, "code without language"},
 	}
 	f := NewCommentFormatter(mockConverter{})
 	for _, tt := range tbl {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.out, f.FormatText(tt.in))
+			raw := strings.HasPrefix(tt.in, `no_smartpants`)
+			t.Logf("raw: %v", raw)
+			assert.Equal(t, tt.out, f.FormatText(tt.in, raw))
 		})
 	}
 }
 
 func TestFormatter_FormatTextNoConverter(t *testing.T) {
 	f := NewCommentFormatter()
-	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatTextConverterFunc(t *testing.T) {
 	fn := CommentConverterFunc(func(text string) string { return "zz!" + text })
 	f := NewCommentFormatter(fn)
-	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatComment(t *testing.T) {
@@ -78,7 +87,7 @@ func TestFormatter_FormatComment(t *testing.T) {
 	f := NewCommentFormatter(mockConverter{})
 	exp := comment
 	exp.Text = "<p>blah</p>\n\n<p>xyz</p>\n!converted"
-	assert.Equal(t, exp, f.Format(comment))
+	assert.Equal(t, exp, f.Format(comment, false))
 }
 
 func TestFormatter_ShortenAutoLinks(t *testing.T) {

backend/app/store/image/bolt_store.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"time"
 
@@ -118,6 +119,18 @@ func (b *Bolt) Load(id string) ([]byte, error) {
 	return data, nil
 }
 
+// Delete image from storage
+func (b *Bolt) Delete(id string) error {
+	return b.db.Update(func(tx *bolt.Tx) error {
+		// deleting a non-existing key doesn't return an error, so joining errors from deleting an image
+		// from both buckets is safe and will return nil if there are no errors on the real delete
+		// or image is absent in both buckets
+		err := tx.Bucket([]byte(imagesBktName)).Delete([]byte(id))
+		err = errors.Join(err, tx.Bucket([]byte(imagesStagedBktName)).Delete([]byte(id)))
+		return err
+	})
+}
+
 // Cleanup runs scan of staging and removes old data based on ttl
 func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error {
 	return b.db.Update(func(tx *bolt.Tx) error {

backend/app/store/image/bolt_store_test.go

@@ -58,6 +58,36 @@ func TestBoltStore_LoadAfterSave(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestBoltStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	// delete image from permanent storage
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// delete staging image
+	id = "staging_img"
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestBoltStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()

backend/app/store/image/fs_store.go

@@ -23,7 +23,8 @@ type FileSystem struct {
 	Staging    string
 	Partitions int
 
-	crc struct {
+	moveLock sync.Mutex // needed only for deleting images or moving them from staging to permanent storage
+	crc      struct {
 		*crc64.Table
 		sync.Once
 		mask    string
@@ -50,6 +51,8 @@ func (f *FileSystem) Save(id string, img []byte) error {
 
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 	log.Printf("[DEBUG] Commit image %s", id)
 	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
 
@@ -107,11 +110,31 @@ func (f *FileSystem) Load(id string) ([]byte, error) {
 	return io.ReadAll(fh)
 }
 
+// Delete image from storage
+func (f *FileSystem) Delete(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
+	staging := f.location(f.Staging, id)
+	// file doesn't exist on staging, delete from permanent location
+	if _, err := os.Stat(staging); os.IsNotExist(err) {
+		file := f.location(f.Location, id)
+		e := os.Remove(file)
+		_ = os.Remove(path.Dir(file)) // try to remove directory
+		return e
+	}
+	// delete file from staging
+	err := os.Remove(staging)
+	_ = os.Remove(path.Dir(staging)) // try to remove directory
+	return err
+}
+
 // Cleanup runs scan of staging and removes old files based on ttl
 func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error {
 	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
 		return nil
 	}
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 
 	// we can ignore context as on local FS remove is relatively fast operation
 	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
@@ -143,7 +166,7 @@ func (f *FileSystem) Info() (StoreInfo, error) {
 	}
 
 	var ts time.Time
-	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+	err := filepath.Walk(f.Staging, func(_ string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}

backend/app/store/image/fs_store_test.go

@@ -131,6 +131,31 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestFsStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// create file on staging
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestFsStore_location(t *testing.T) {
 	tbl := []struct {
 		partitions int

backend/app/store/image/image.go

@@ -70,6 +70,7 @@ type Store interface {
 	Info() (StoreInfo, error)         // get meta information about storage
 	Save(id string, img []byte) error // store image with passed id to staging
 	Load(id string) ([]byte, error)   // load image by ID
+	Delete(id string) error           // delete image by ID
 
 	ResetCleanupTimer(id string) error                    // resets cleanup timer for the image, called on comment preview
 	Commit(id string) error                               // move image from staging to permanent
@@ -88,8 +89,8 @@ func NewService(s Store, p ServiceParams) *Service {
 	return &Service{ServiceParams: p, store: s}
 }
 
-// SubmitAndCommit multiple ids immediately
-func (s *Service) SubmitAndCommit(idsFn func() []string) error {
+// Commit multiple ids immediately
+func (s *Service) Commit(idsFn func() []string) error {
 	errs := new(multierror.Error)
 	for _, id := range idsFn() {
 		err := s.store.Commit(id)
@@ -117,7 +118,7 @@ func (s *Service) Submit(idsFn func() []string) {
 				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.EditDuration {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
-				err := s.SubmitAndCommit(req.idsFn)
+				err := s.Commit(req.idsFn)
 				if err != nil {
 					log.Printf("[WARN] image commit error %v", err)
 				}
@@ -141,39 +142,14 @@ func (s *Service) Submit(idsFn func() []string) {
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
 func (s *Service) ExtractPictures(commentHTML string) (ids []string) {
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
-	if err != nil {
-		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
-		return nil
-	}
-	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
-		if im, ok := sl.Attr("src"); ok {
-			if strings.Contains(im, s.ImageAPI) {
-				elems := strings.Split(im, "/")
-				if len(elems) >= 2 {
-					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
-					ids = append(ids, id)
-				}
-			}
-			if strings.Contains(im, s.ProxyAPI) {
-				proxiedURL, err := url.Parse(im)
-				if err != nil {
-					return
-				}
-				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
-				if err != nil {
-					return
-				}
-				imgID, err := CachedImgID(string(imgURL))
-				if err != nil {
-					return
-				}
-				ids = append(ids, imgID)
-			}
-		}
-	})
+	return s.extractImageIDs(commentHTML, true)
+}
 
-	return ids
+// ExtractNonProxiedPictures gets list of non-proxied images from the doc html and convert from urls to ids, i.e. user/pic.png
+// This method is used in image check on post preview and load, as proxied images have lazy loading
+// and wouldn't be present on disk but still valid as they will be loaded the first time someone requests them.
+func (s *Service) ExtractNonProxiedPictures(commentHTML string) (ids []string) {
+	return s.extractImageIDs(commentHTML, false)
 }
 
 // Cleanup runs periodic cleanup with 1.5*ServiceParams.EditDuration. Blocking loop, should be called inside of goroutine by consumer
@@ -237,6 +213,11 @@ func (s *Service) Load(id string) ([]byte, error) {
 	return s.store.Load(id)
 }
 
+// Delete wraps storage Delete function.
+func (s *Service) Delete(id string) error {
+	return s.store.Delete(id)
+}
+
 // Save wraps storage Save function, validating and resizing the image before calling it.
 func (s *Service) Save(userID string, r io.Reader) (id string, err error) {
 	id = path.Join(userID, guid())
@@ -262,6 +243,43 @@ func (s *Service) ImgContentType(img []byte) string {
 	return contentType
 }
 
+// returns list of image IDs from the comment html, including proxied images if includeProxied is true
+func (s *Service) extractImageIDs(commentHTML string, includeProxied bool) (ids []string) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
+		return nil
+	}
+	doc.Find("img").Each(func(_ int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					ids = append(ids, id)
+				}
+			}
+			if includeProxied && strings.Contains(im, s.ProxyAPI) {
+				proxiedURL, err := url.Parse(im)
+				if err != nil {
+					return
+				}
+				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
+				if err != nil {
+					return
+				}
+				imgID, err := CachedImgID(string(imgURL))
+				if err != nil {
+					return
+				}
+				ids = append(ids, imgID)
+			}
+		}
+	})
+
+	return ids
+}
+
 // prepareImage calls readAndValidateImage and resize on provided image.
 func (s *Service) prepareImage(r io.Reader) ([]byte, error) {
 	data, err := readAndValidateImage(r, s.MaxSize)

backend/app/store/image/image_mock.go

@@ -4,9 +4,9 @@
 package image
 
 import (
-	context "context"
+	"context"
 	"sync"
-	time "time"
+	"time"
 )
 
 // Ensure, that StoreMock does implement Store.
@@ -25,6 +25,9 @@ var _ Store = &StoreMock{}
 //			CommitFunc: func(id string) error {
 //				panic("mock out the Commit method")
 //			},
+//			DeleteFunc: func(id string) error {
+//				panic("mock out the Delete method")
+//			},
 //			InfoFunc: func() (StoreInfo, error) {
 //				panic("mock out the Info method")
 //			},
@@ -50,6 +53,9 @@ type StoreMock struct {
 	// CommitFunc mocks the Commit method.
 	CommitFunc func(id string) error
 
+	// DeleteFunc mocks the Delete method.
+	DeleteFunc func(id string) error
+
 	// InfoFunc mocks the Info method.
 	InfoFunc func() (StoreInfo, error)
 
@@ -76,6 +82,11 @@ type StoreMock struct {
 			// ID is the id argument value.
 			ID string
 		}
+		// Delete holds details about calls to the Delete method.
+		Delete []struct {
+			// ID is the id argument value.
+			ID string
+		}
 		// Info holds details about calls to the Info method.
 		Info []struct {
 		}
@@ -99,6 +110,7 @@ type StoreMock struct {
 	}
 	lockCleanup           sync.RWMutex
 	lockCommit            sync.RWMutex
+	lockDelete            sync.RWMutex
 	lockInfo              sync.RWMutex
 	lockLoad              sync.RWMutex
 	lockResetCleanupTimer sync.RWMutex
@@ -173,6 +185,38 @@ func (mock *StoreMock) CommitCalls() []struct {
 	return calls
 }
 
+// Delete calls DeleteFunc.
+func (mock *StoreMock) Delete(id string) error {
+	if mock.DeleteFunc == nil {
+		panic("StoreMock.DeleteFunc: method is nil but Store.Delete was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockDelete.Lock()
+	mock.calls.Delete = append(mock.calls.Delete, callInfo)
+	mock.lockDelete.Unlock()
+	return mock.DeleteFunc(id)
+}
+
+// DeleteCalls gets all the calls that were made to Delete.
+// Check the length with:
+//
+//	len(mockedStore.DeleteCalls())
+func (mock *StoreMock) DeleteCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockDelete.RLock()
+	calls = mock.calls.Delete
+	mock.lockDelete.RUnlock()
+	return calls
+}
+
 // Info calls InfoFunc.
 func (mock *StoreMock) Info() (StoreInfo, error) {
 	if mock.InfoFunc == nil {

backend/app/store/image/image_test.go

@@ -19,10 +19,10 @@ import (
 
 func TestService_SaveAndLoad(t *testing.T) {
 	store := StoreMock{
-		SaveFunc: func(id string, img []byte) error {
+		SaveFunc: func(string, []byte) error {
 			return nil
 		},
-		LoadFunc: func(id string) ([]byte, error) {
+		LoadFunc: func(string) ([]byte, error) {
 			return nil, nil
 		},
 	}
@@ -105,6 +105,7 @@ func TestService_ExtractPictures(t *testing.T) {
 	ids = svc.ExtractPictures(html)
 	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
+	require.Empty(t, svc.ExtractNonProxiedPictures(html), "no non-proxied images expected to be found")
 
 	// bad url
 	html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
@@ -125,7 +126,7 @@ func TestService_ExtractPictures(t *testing.T) {
 
 func TestService_Cleanup(t *testing.T) {
 	store := StoreMock{
-		CleanupFunc: func(ctx context.Context, ttl time.Duration) error {
+		CleanupFunc: func(context.Context, time.Duration) error {
 			return nil
 		},
 	}
@@ -140,17 +141,13 @@ func TestService_Cleanup(t *testing.T) {
 
 func TestService_Submit(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
-			return nil
-		},
-		ResetCleanupTimerFunc: func(id string) error {
-			return nil
-		},
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	svc := NewService(&store, ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100})
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	assert.Equal(t, 3, len(store.ResetCleanupTimerCalls()))
-	err := svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} })
+	err := svc.Commit(func() []string { return []string{"id4", "id5"} })
 	assert.NoError(t, err)
 	svc.Submit(func() []string { return []string{"id6", "id7"} })
 	assert.Equal(t, 5, len(store.ResetCleanupTimerCalls()))
@@ -163,12 +160,8 @@ func TestService_Submit(t *testing.T) {
 
 func TestService_Close(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
-			return nil
-		},
-		ResetCleanupTimerFunc: func(id string) error {
-			return nil
-		},
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
@@ -181,10 +174,8 @@ func TestService_Close(t *testing.T) {
 
 func TestService_SubmitDelay(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
-			return nil
-		},
-		ResetCleanupTimerFunc: func(id string) error {
+		CommitFunc: func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error {
 			return nil
 		},
 	}
@@ -286,3 +277,11 @@ func TestCachedImgID(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "cached_images/"+Sha1Str("example.org")+"-"+Sha1Str(imgURL), img)
 }
+
+func TestService_DoubleClose(*testing.T) {
+	store := StoreMock{}
+	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
+	svc.Close(context.TODO())
+	// second call should not result in panic
+	svc.Close(context.TODO())
+}

backend/app/store/image/remote_store.go

@@ -41,6 +41,12 @@ func (r *RPC) Load(id string) ([]byte, error) {
 	return io.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg)))
 }
 
+// Delete image from storage
+func (r *RPC) Delete(id string) error {
+	_, err := r.Call("image.delete", id)
+	return err
+}
+
 // Commit file stored in staging location by moving it to permanent location
 func (r *RPC) Commit(id string) error {
 	_, err := r.Call("image.commit", id)

backend/app/store/image/remote_store_test.go

@@ -41,6 +41,18 @@ func TestRemote_Load(t *testing.T) {
 	assert.Equal(t, gopherPNGBytes(), res)
 }
 
+func TestRemote_Delete(t *testing.T) {
+	ts := testServer(t, `{"method":"image.delete","params":"54321","id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.Delete("54321")
+	assert.NoError(t, err)
+}
+
 func TestRemote_Commit(t *testing.T) {
 	ts := testServer(t, `{"method":"image.commit","params":"gopher_id","id":1}`, `{"id":1}`)
 	defer ts.Close()

backend/app/store/service/service.go

@@ -5,15 +5,17 @@ package service
 import (
 	"fmt"
 	"math"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-multierror"
+	bf "github.com/russross/blackfriday/v2"
 
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/admin"
@@ -26,6 +28,7 @@ type DataStore struct {
 	Engine              engine.Interface
 	EditDuration        time.Duration
 	AdminStore          admin.Store
+	MinCommentSize      int
 	MaxCommentSize      int
 	MaxVotes            int
 	RestrictSameIPVotes struct {
@@ -46,7 +49,7 @@ type DataStore struct {
 	}
 
 	repliesCache struct {
-		lcw.LoadingCache
+		lcw.LoadingCache[struct{}]
 		once sync.Once
 	}
 }
@@ -68,7 +71,7 @@ type PostMetaData struct {
 	ReadOnly bool   `json:"read_only"`
 }
 
-const defaultCommentMaxSize = 2000
+const defaultCommentMaxSize = 2048
 const maxLastCommentsReply = 5000
 
 // UnlimitedVotes doesn't restrict MaxVotes
@@ -278,7 +281,7 @@ func (s *DataStore) submitImages(comment store.Comment) {
 
 	var err error
 	if comment.Imported {
-		err = s.ImageService.SubmitAndCommit(idsFn)
+		err = s.ImageService.Commit(idsFn)
 	} else {
 		s.ImageService.Submit(idsFn)
 	}
@@ -531,7 +534,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 func (s *DataStore) HasReplies(comment store.Comment) bool {
 	s.repliesCache.once.Do(func() {
 		// default expiration time of 5 minutes and cleanup time of 2.5 minutes
-		s.repliesCache.LoadingCache, _ = lcw.NewExpirableCache(lcw.TTL(5 * time.Minute))
+		o := lcw.NewOpts[struct{}]()
+		s.repliesCache.LoadingCache, _ = lcw.NewExpirableCache[struct{}](o.TTL(5 * time.Minute))
 	})
 
 	if _, found := s.repliesCache.Peek(comment.ID); found {
@@ -551,7 +555,7 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 				// When this code is reached, key "comment.ID" is not in cache.
 				// Calling cache.Get on it will put it in cache with 5 minutes TTL.
 				// We call it with empty struct as value as we care about keys and not values.
-				_, _ = s.repliesCache.Get(comment.ID, func() (interface{}, error) { return struct{}{}, nil })
+				_, _ = s.repliesCache.Get(comment.ID, func() (struct{}, error) { return struct{}{}, nil })
 				return true
 			}
 		}
@@ -627,7 +631,9 @@ func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, e
 	return res, nil
 }
 
-// ValidateComment checks if comment size below max and user fields set
+// ValidateComment checks if comment size below max and user fields set.
+// It also validates the absence of relative links as they are almost never the intention of the commenter,
+// usually added by mistakes and only create confusion.
 func (s *DataStore) ValidateComment(c *store.Comment) error {
 	maxSize := s.MaxCommentSize
 	if s.MaxCommentSize <= 0 {
@@ -636,13 +642,33 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
 	if c.Orig == "" {
 		return fmt.Errorf("empty comment text")
 	}
-	if len([]rune(c.Orig)) > maxSize {
-		return fmt.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, len([]rune(c.Orig)))
+	commentLength := len([]rune(c.Orig))
+	if commentLength > maxSize {
+		return fmt.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, commentLength)
+	}
+	if s.MinCommentSize > 0 && commentLength < s.MinCommentSize {
+		return fmt.Errorf("comment text is smaller than min allowed size %d (%d)", s.MinCommentSize, commentLength)
 	}
 	if c.User.ID == "" || c.User.Name == "" {
 		return fmt.Errorf("empty user info")
 	}
-	return nil
+
+	// for validation purposes it's not important if SmartyPants formatting is disabled or enabled,
+	// while for storing the comment that flag is set based on user preference
+	mdExt, rend := store.GetMdExtensionsAndRenderer(false)
+	parser := bf.New(bf.WithRenderer(rend), bf.WithExtensions(bf.CommonExtensions), bf.WithExtensions(mdExt))
+	var wrongLinkError error
+	parser.Parse([]byte(c.Orig)).Walk(func(node *bf.Node, _ bool) bf.WalkStatus {
+		if len(node.LinkData.Destination) != 0 &&
+			!(strings.HasPrefix(string(node.LinkData.Destination), "http://") ||
+				strings.HasPrefix(string(node.LinkData.Destination), "https://") ||
+				strings.HasPrefix(string(node.LinkData.Destination), "mailto:")) {
+			wrongLinkError = fmt.Errorf("links should start with mailto:, http:// or https://")
+			return bf.Terminate
+		}
+		return bf.GoToNext
+	})
+	return wrongLinkError
 }
 
 // IsAdmin checks if usesID in the list of admins
@@ -737,7 +763,23 @@ func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo
 	if len(res) == 0 {
 		return store.PostInfo{}, fmt.Errorf("post %+v not found", locator)
 	}
-	return res[0], nil
+	// URL request
+	if locator.URL != "" {
+		return res[0], nil
+	}
+	// site-wide request which returned multiple store.PostInfo, so that URL and ReadOnly flags don't make sense
+	var info store.PostInfo
+	for _, i := range res {
+		info.Count += i.Count
+		if info.FirstTS.IsZero() || i.FirstTS.Before(info.FirstTS) {
+			info.FirstTS = i.FirstTS
+		}
+		if info.LastTS.IsZero() || i.LastTS.After(info.LastTS) {
+			info.LastTS = i.LastTS
+		}
+	}
+	return info, nil
+
 }
 
 // Delete comment by id
@@ -756,6 +798,34 @@ func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.D
 		s.repliesCache.Delete(commentID)
 		s.repliesCache.Delete(comment.ParentID)
 	}
+
+	// delete images from the comment if they are not reused elsewhere in comments to the same page
+	idsFn := func() []string { // get IDs of all images from the same URL to verify if image from deleted comment was reused
+		comments, e := s.Engine.Find(engine.FindRequest{Locator: locator})
+		if e != nil {
+			log.Printf("[WARN] can't get comments %s text for deleted comment image check, %v", comment.ID, err)
+			return nil
+		}
+		var imgIDs = []string{}
+		for _, cc := range comments {
+			// exclude the comment we are deleting
+			if cc.ID != commentID {
+				imgIDs = append(imgIDs, s.ImageService.ExtractPictures(cc.Text)...)
+			}
+		}
+		return imgIDs
+	}
+	commentImgIDs := s.ImageService.ExtractPictures(comment.Text)
+	pageImgIDs := idsFn()
+	for _, id := range commentImgIDs {
+		if !slices.Contains(pageImgIDs, id) {
+			if err := s.ImageService.Delete(id); err != nil {
+				log.Printf("[WARN] failed to delete image %s on comment %s deletion, %v", id, commentID, err)
+			}
+		}
+	}
+	log.Printf("[ERROR] commentImgIDs: %v, pageImgIDs: %v", commentImgIDs, pageImgIDs)
+
 	req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
 	return s.Engine.Delete(req)
 }

backend/app/store/service/service_test.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -131,7 +132,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
 	b := DataStore{Engine: eng, AdminStore: ks,
-		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})}
 	defer b.Close()
 
 	postPath := "/post/42"
@@ -171,6 +172,50 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
 }
 
+func TestService_Put(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, AdminStore: ks}
+
+	comment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "test text",
+		User:      store.User{ID: "user2", Name: "user name 2"},
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+	}
+	_, err := b.Create(comment)
+	require.NoError(t, err)
+
+	// create new comment with everything different to replace the first one with fields below
+	updatedComment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-new",
+		Text:      "new text",
+		User:      store.User{ID: "user3", Name: "user name 3"},
+		Locator:   store.Locator{URL: "https://example.com", SiteID: "example"},
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+	}
+
+	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, updatedComment)
+	require.NoError(t, err)
+
+	// request with wrong user, should not affect the comment user
+	got, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", store.User{ID: "user1", Name: "user name 1"})
+	require.NoError(t, err)
+	assert.Equal(t, "c-1", got.ID)
+	assert.Equal(t, "new text", got.Text)
+	assert.Equal(t, "id-1", got.ParentID, "should be unaltered")
+	assert.Equal(t, "user2", got.User.ID, "should be unaltered")
+	assert.Equal(t, "user name 2", got.User.Name, "should be unaltered")
+	assert.Equal(t, "https://radio-t.com", got.Locator.URL, "should be unaltered")
+	assert.Equal(t, "radio-t", got.Locator.SiteID, "should be unaltered")
+	assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), got.Timestamp, "should be unaltered")
+
+}
+
 func TestService_SetTitle(t *testing.T) {
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -195,7 +240,7 @@ func TestService_SetTitle(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
 	b := DataStore{Engine: eng, AdminStore: ks,
-		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})}
 	defer b.Close()
 	comment := store.Comment{
 		Text:      "text",
@@ -413,7 +458,6 @@ func TestService_VoteAggressive(t *testing.T) {
 	assert.Equal(t, 0, len(res[0].VotedIPs), "vote ips hidden")
 
 	// random +1/-1 result should be [0..2]
-	rand.Seed(time.Now().UnixNano())
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
 		go func() {
@@ -546,6 +590,111 @@ func TestService_VoteControversy(t *testing.T) {
 	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
 }
 
+func TestService_RestrictedWords(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	restictedWordLister := StaticRestrictedWordsLister{Words: []string{"restricted"}}
+	b := DataStore{Engine: eng, AdminStore: ks, RestrictedWordsMatcher: NewRestrictedWordsMatcher(restictedWordLister)}
+
+	// test creating a comment with restricted words which should fail with appropriate error
+	reply := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "restricted word",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+	id, err := b.Create(reply)
+	assert.EqualError(t, err, ErrRestrictedWordsFound.Error(), "should fail with RestrictedWordError")
+	assert.Empty(t, id)
+}
+
+func TestDataStore_AdminStoreErrors(t *testing.T) {
+	badKey := true
+	badEnabled := true
+	as := admin.StoreMock{
+		OnEventFunc: func(string, admin.EventType) error { return errors.New("err") },
+		KeyFunc: func(string) (string, error) {
+			if badKey {
+				return "", errors.New("mock key err")
+			}
+			return "secret", nil
+		},
+		EnabledFunc: func(string) (bool, error) {
+			if badEnabled {
+				return false, errors.New("mock enabled err")
+			}
+			return true, nil
+		},
+		AdminsFunc: func(string) ([]string, error) { return nil, errors.New("mock admins err") },
+	}
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, AdminStore: &as, MaxVotes: -1}
+	comment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "restricted word",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+
+	// Key call error
+	id, err := b.Create(comment)
+	assert.ErrorContainsf(t, err, "mock key err", "should fail with mock error")
+	assert.Empty(t, id)
+	assert.Equal(t, len(as.KeyCalls()), 1)
+	assert.Equal(t, len(as.EnabledCalls()), 0)
+	assert.Equal(t, len(as.OnEventCalls()), 0)
+
+	// Enabled call error
+	badKey = false
+	id, err = b.Create(comment)
+	assert.ErrorContains(t, err, "mock enabled err", "should fail with mock error")
+	assert.Empty(t, id)
+	assert.Equal(t, len(as.KeyCalls()), 2)
+	assert.Equal(t, len(as.EnabledCalls()), 1)
+	assert.Equal(t, len(as.OnEventCalls()), 0)
+
+	// only OnEvent error
+	badEnabled = false
+	id, err = b.Create(comment)
+	assert.NoError(t, err, "OnEvent error should be just logged")
+	assert.Equal(t, id, "c-1")
+	assert.Equal(t, len(as.KeyCalls()), 3)
+	assert.Equal(t, len(as.EnabledCalls()), 2)
+	assert.Equal(t, len(as.OnEventCalls()), 1)
+
+	// OnEvent error on Vote call
+	_, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "c-1",
+		UserID: "user4", Val: true})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+	assert.Equal(t, len(as.KeyCalls()), 4)
+	assert.Equal(t, len(as.EnabledCalls()), 3)
+	assert.Equal(t, len(as.OnEventCalls()), 2)
+
+	// Admins error
+	isAdmin := b.IsAdmin("radio-t", "user2")
+	assert.False(t, isAdmin)
+	assert.Equal(t, len(as.AdminsCalls()), 1)
+	assert.Equal(t, len(as.OnEventCalls()), 2)
+
+	// OnEvent error on EditComment call
+	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", EditRequest{Text: "new text"})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+
+	// OnEvent error on Delete call
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", store.SoftDelete)
+	assert.NoError(t, err, "OnEvent error should be just logged")
+
+	// OnEvent error on EditComment Delete call
+	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", EditRequest{Delete: true})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+}
+
 func TestService_VoteSameIP(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
@@ -775,27 +924,31 @@ func TestService_EditCommentAdmin(t *testing.T) {
 }
 
 func TestService_ValidateComment(t *testing.T) {
-	b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{MinCommentSize: 6, MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
 	longText := fmt.Sprintf("%4000s", "X")
 
 	tbl := []struct {
 		inp store.Comment
-		err error
+		err string
 	}{
-		{inp: store.Comment{}, err: fmt.Errorf("empty comment text")},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: nil},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: fmt.Errorf("empty user info")},
-		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: fmt.Errorf("comment text exceeded max allowed size 2000 (4000)")},
+		{inp: store.Comment{}, err: "empty comment text"},
+		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: ""},
+		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: "empty user info"},
+		{inp: store.Comment{Orig: "short", User: store.User{ID: "myid", Name: "name"}}, err: "comment text is smaller than min allowed size 6 (5)"},
+		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: "comment text exceeded max allowed size 2000 (4000)"},
+		{inp: store.Comment{Orig: "here is a link with relative URL: [google.com](url)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
+		{inp: store.Comment{Orig: "here is a link with relative URL: [google.com](url)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
+		{inp: store.Comment{Orig: "multiple links, one is bad: [test](http://test) [test2](bad_url) [test3](https://test3)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
 	}
 
 	for n, tt := range tbl {
 		err := b.ValidateComment(&tt.inp)
-		if tt.err == nil {
+		if tt.err == "" {
 			assert.NoError(t, err, "check #%d", n)
 			continue
 		}
 		require.Error(t, err)
-		assert.EqualError(t, tt.err, err.Error(), "check #%d", n)
+		assert.EqualError(t, err, tt.err, "check #%d", n)
 	}
 }
 
@@ -1154,7 +1307,7 @@ func TestService_Find(t *testing.T) {
 	assert.InDelta(t, 0, res[1].Controversy, 0.01)
 
 	// make sure title sanitized
-	assert.Equal(t, "some title, &lt;a href=\"http://radio-t.com\" rel=\"nofollow\"&gt;link&lt;/a&gt;", res[0].PostTitle)
+	assert.Equal(t, "some title, link", res[0].PostTitle)
 }
 
 func TestService_FindSince(t *testing.T) {
@@ -1183,18 +1336,55 @@ func TestService_Info(t *testing.T) {
 	b := DataStore{Engine: eng, EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", nil, []string{"user2"}, "user@email.com")}
 
-	info, err := b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
+	// add one more comment for another URL to test site-wide Info request
+	comment := store.Comment{
+		ID:        "123456xyz",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/another", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	require.NoError(t, err)
+
+	// get non-existing URL info
+	info, err := b.Info(store.Locator{URL: "bad", SiteID: "radio-t"}, 0)
+	assert.Error(t, err)
+	assert.Empty(t, info)
+
+	// get non-existing site info
+	info, err = b.Info(store.Locator{SiteID: "bad"}, 0)
+	assert.Error(t, err)
+	assert.Empty(t, info)
+
+	// test two initially created comments and store first comment FirstTS
+	info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
 	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com", info.URL)
 	assert.Equal(t, 2, info.Count)
 	assert.False(t, info.ReadOnly)
 	assert.True(t, info.LastTS.After(info.FirstTS))
+	firstTS := info.FirstTS
 
 	time.Sleep(1 * time.Second) // make post RO in 1sec
 	info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 1)
 	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com", info.URL)
 	assert.True(t, info.ReadOnly)
+
+	// get last created comment LastTS
+	info, err = b.Info(store.Locator{URL: "https://radio-t.com/another", SiteID: "radio-t"}, 0)
+	require.NoError(t, err)
+	lastTS := info.LastTS
+
+	// site-level request
+	info, err = b.Info(store.Locator{SiteID: "radio-t"}, 1)
+	require.NoError(t, err)
+	assert.Equal(t, 3, info.Count)
+	assert.Empty(t, info.URL, "site-level request should not set URL")
+	assert.False(t, info.ReadOnly, "site-level request should not set ReadOnly")
+	assert.Equal(t, firstTS, info.FirstTS, "site-level request should have FirstTS from the first post")
+	assert.Equal(t, lastTS, info.LastTS, "site-level request should have LastTS from the last post")
 }
 
 func TestService_Delete(t *testing.T) {
@@ -1216,6 +1406,80 @@ func TestService_Delete(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestService_deleteImagesOnCommentDelete(t *testing.T) {
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	mockStore := image.StoreMock{
+		DeleteFunc:            func(string) error { return nil },
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
+	}
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 50 * time.Millisecond,
+			ImageAPI:     "/images/dev/",
+			ProxyAPI:     "/non_existent",
+		})
+	defer imgSvc.Close(context.TODO())
+
+	// two comments for https://radio-t.com
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+	b.submitImages(c)
+	// reply to the first comment with one new image and one existing one
+	c = store.Comment{
+		ID:       "id-23",
+		ParentID: "id-22",
+		Text:     `some text <img src="/images/dev/pic2.png"/> xx <img src="/images/dev/pic3.png"/>`,
+		Locator:  store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:     store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Engine.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+	b.submitImages(c)
+
+	// verify that images are in staging store
+	assert.Equal(t, 4, len(mockStore.ResetCleanupTimerCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.ResetCleanupTimerCalls()[0].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.ResetCleanupTimerCalls()[1].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.ResetCleanupTimerCalls()[2].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.ResetCleanupTimerCalls()[3].ID)
+	time.Sleep(b.EditDuration + 100*time.Millisecond)
+	// verify that they got into the main store
+	assert.Equal(t, 4, len(mockStore.CommitCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.CommitCalls()[0].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.CommitCalls()[1].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.CommitCalls()[2].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.CommitCalls()[3].ID)
+
+	// delete the first comment
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-22", store.SoftDelete)
+	assert.NoError(t, err)
+	// verify that images are deleted from the main store
+	assert.Equal(t, 1, len(mockStore.DeleteCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.DeleteCalls()[0].ID)
+
+	// delete the second comment
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-23", store.SoftDelete)
+	assert.NoError(t, err)
+	// verify that images are deleted from the main store
+	assert.Equal(t, 3, len(mockStore.DeleteCalls()))
+	assert.Equal(t, "dev/pic2.png", mockStore.DeleteCalls()[1].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.DeleteCalls()[2].ID)
+}
+
 // DeleteUser removes all comments from user
 func TestService_DeleteUser(t *testing.T) {
 	// two comments for https://radio-t.com, no reply
@@ -1396,8 +1660,8 @@ func TestService_submitImages(t *testing.T) {
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
 	mockStore := image.StoreMock{
-		CommitFunc:            func(id string) error { return nil },
-		ResetCleanupTimerFunc: func(id string) error { return nil },
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	imgSvc := image.NewService(&mockStore,
 		image.ServiceParams{
@@ -1438,10 +1702,10 @@ func TestService_ResubmitStagingImages(t *testing.T) {
 		InfoFunc: func() (image.StoreInfo, error) {
 			return image.StoreInfo{FirstStagingImageTS: time.Time{}.Add(time.Second)}, nil
 		},
-		CommitFunc: func(id string) error {
+		CommitFunc: func(string) error {
 			return nil
 		},
-		ResetCleanupTimerFunc: func(id string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	imgSvc := image.NewService(&mockStore,
 		image.ServiceParams{
@@ -1533,7 +1797,7 @@ func TestService_ResubmitStagingImages_EngineError(t *testing.T) {
 
 	first := true
 	engineMock := engine.InterfaceMock{
-		FindFunc: func(req engine.FindRequest) ([]store.Comment, error) {
+		FindFunc: func(engine.FindRequest) ([]store.Comment, error) {
 			if first {
 				first = false
 				return nil, nil
@@ -1558,7 +1822,7 @@ func TestService_ResubmitStagingImages_EngineError(t *testing.T) {
 
 func TestService_alterComment(t *testing.T) {
 	engineMock := engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			return false, nil
 		},
 	}
@@ -1580,7 +1844,7 @@ func TestService_alterComment(t *testing.T) {
 
 	first := true
 	engineMock = engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			if first {
 				first = false
 				return false, nil
@@ -1598,7 +1862,7 @@ func TestService_alterComment(t *testing.T) {
 
 	first = true
 	engineMock = engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			if first {
 				first = false
 				return true, nil
@@ -1638,6 +1902,30 @@ func Benchmark_ServiceCreate(b *testing.B) {
 	}
 }
 
+func TestService_DoubleClose_Bolt(t *testing.T) {
+	dbFile := fmt.Sprintf("%s/test-remark42-%d.db", os.TempDir(), rand.Intn(9999999999))
+	defer func() { _ = os.Remove(dbFile) }()
+
+	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: dbFile, SiteID: "radio-t"})
+	svc := DataStore{Engine: boltStore, EditDuration: 50 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
+	require.NoError(t, err)
+	assert.NoError(t, boltStore.Close())
+	assert.NoError(t, boltStore.Close(), "second call should not result in panic or errors")
+	assert.NoError(t, svc.Close())
+	assert.NoError(t, svc.Close(), "second call should not result in panic or errors")
+}
+
+func TestService_DoubleClose_Static(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, AdminStore: ks,
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})}
+	assert.NoError(t, b.Close())
+	// second call should not result in panic or errors
+	assert.NoError(t, b.Close())
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) (e engine.Interface, teardown func()) {
 	testDBLoc, err := os.MkdirTemp("", "test_image_r42")

backend/app/store/service/testdata/tree.json

@@ -242,11 +242,5 @@
                   "time": "2017-12-25T19:47:22Z"
               }
           }
-      ],
-      "info": {
-          "url": "url",
-          "count": 12,
-          "first_time": "2017-12-25T19:46:01Z",
-          "last_time": "2017-12-25T19:47:22Z"
-      }
-  }
+      ]
+  }

backend/app/store/service/testdata/tree_del.json

@@ -176,11 +176,5 @@
                  }
              }]
          }]
-     }],
-     "info": {
-         "url": "url",
-         "count": 8,
-         "first_time": "2017-12-25T19:46:01Z",
-         "last_time": "2017-12-25T19:47:05Z"
-     }
- }
+     }]
+ }

backend/app/store/service/title.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"golang.org/x/net/html"
 )
@@ -19,32 +20,53 @@ const (
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
-	client http.Client
-	cache  lcw.LoadingCache
+	client         http.Client
+	cache          lcw.LoadingCache[string]
+	allowedDomains []string
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
-func NewTitleExtractor(client http.Client) *TitleExtractor {
+func NewTitleExtractor(client http.Client, allowedDomains []string) *TitleExtractor {
+	log.Printf("[DEBUG] creating extractor, allowed domains %+v", allowedDomains)
 	res := TitleExtractor{
-		client: client,
+		client:         client,
+		allowedDomains: allowedDomains,
 	}
 	var err error
-	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
+	o := lcw.NewOpts[string]()
+	res.cache, err = lcw.NewExpirableCache(o.TTL(teCacheTTL), o.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
 		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
-		res.cache = &lcw.Nop{}
+		res.cache = &lcw.Nop[string]{}
 	}
 	return &res
 }
 
 // Get page for url and return title
-func (t *TitleExtractor) Get(url string) (string, error) {
+func (t *TitleExtractor) Get(pageURL string) (string, error) {
+	// parse domain of the URL and check if it's in the allowed list
+	u, err := url.Parse(pageURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse url %s: %w", pageURL, err)
+	}
+	allowed := false
+	for _, domain := range t.allowedDomains {
+		if u.Hostname() == domain ||
+			(strings.HasSuffix(u.Hostname(), domain) && // suffix match, e.g. "example.com" matches "www.example.com"
+				u.Hostname()[len(u.Hostname())-len(domain)-1] == '.') { // but we should not match "notexample.com"
+			allowed = true
+			break
+		}
+	}
+	if !allowed {
+		return "", fmt.Errorf("domain %s is not allowed", u.Host)
+	}
 	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
 	defer client.CloseIdleConnections()
-	b, err := t.cache.Get(url, func() (interface{}, error) {
-		resp, err := client.Get(url)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load page %s: %w", url, err)
+	b, err := t.cache.Get(pageURL, func() (string, error) {
+		resp, e := client.Get(pageURL)
+		if e != nil {
+			return "", fmt.Errorf("failed to load page %s: %w", pageURL, e)
 		}
 		defer func() {
 			if err = resp.Body.Close(); err != nil {
@@ -52,23 +74,23 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 			}
 		}()
 		if resp.StatusCode != 200 {
-			return nil, fmt.Errorf("can't load page %s, code %d", url, resp.StatusCode)
+			return "", fmt.Errorf("can't load page %s, code %d", pageURL, resp.StatusCode)
 		}
 
 		title, ok := t.getTitle(resp.Body)
 		if !ok {
-			return nil, fmt.Errorf("can't get title for %s", url)
+			return "", fmt.Errorf("can't get title for %s", pageURL)
 		}
 		return title, nil
 	})
 
 	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
-		_, _ = t.cache.Get(url, func() (interface{}, error) { return "", nil })
+		_, _ = t.cache.Get(pageURL, func() (string, error) { return "", nil })
 		return "", err
 	}
 
-	return b.(string), nil
+	return b, nil
 }
 
 // Close title extractor

backend/app/store/service/title_test.go

@@ -28,7 +28,7 @@ func TestTitle_GetTitle(t *testing.T) {
 		{`<html><body> 2222</body></html>`, false, ""},
 	}
 
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})
 	defer ex.Close()
 	for i, tt := range tbl {
 		tt := tt
@@ -41,7 +41,7 @@ func TestTitle_GetTitle(t *testing.T) {
 }
 
 func TestTitle_Get(t *testing.T) {
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -75,7 +75,7 @@ func TestTitle_GetConcurrent(t *testing.T) {
 	for n := 0; n < 1000; n++ {
 		body += "something something blah blah\n"
 	}
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -104,10 +104,10 @@ func TestTitle_GetConcurrent(t *testing.T) {
 }
 
 func TestTitle_GetFailed(t *testing.T) {
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		atomic.AddInt32(&hits, 1)
 		w.WriteHeader(404)
 	}))
@@ -123,3 +123,10 @@ func TestTitle_GetFailed(t *testing.T) {
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
 }
+
+func TestTitle_DoubleClosed(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})
+	assert.NoError(t, ex.Close())
+	// second call should not result in panic
+	assert.NoError(t, ex.Close())
+}

backend/app/store/service/tree.go

@@ -10,8 +10,7 @@ import (
 
 // Tree is formatter making tree from the list of comments
 type Tree struct {
-	Nodes []*Node        `json:"comments"`
-	Info  store.PostInfo `json:"info,omitempty"`
+	Nodes []*Node `json:"comments"`
 }
 
 // Node is a comment with optional replies
@@ -30,22 +29,12 @@ type recurData struct {
 }
 
 // MakeTree gets unsorted list of comments and produces Tree
-// It will make store.PostInfo by itself and will mark Info.ReadOnly based on passed readOnlyAge
-// Tree maker is local and has no access to the data store. By this reason it has to make Info and won't be able
-// to handle store's read-only status. This status should be set by caller.
-func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree {
+func MakeTree(comments []store.Comment, sortType string) *Tree {
 	if len(comments) == 0 {
 		return &Tree{}
 	}
 
-	res := Tree{
-		Info: store.PostInfo{
-			URL:     comments[0].Locator.URL,
-			FirstTS: comments[0].Timestamp,
-			LastTS:  comments[0].Timestamp,
-		},
-	}
-	res.Info.Count = len(res.filter(comments, func(c store.Comment) bool { return !c.Deleted }))
+	res := Tree{}
 
 	topComments := res.filter(comments, func(c store.Comment) bool { return c.ParentID == "" })
 
@@ -54,23 +43,12 @@ func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree
 		node := Node{Comment: rootComment}
 
 		rd := recurData{}
-		commentsTree, tsModified, tsCreated := res.proc(comments, &node, &rd, rootComment.ID)
-		// skip deleted with no sub-comments ar all sub-comments deleted
+		commentsTree := res.proc(comments, &node, &rd, rootComment.ID)
+		// skip deleted with no sub-comments and all sub-comments deleted
 		if rootComment.Deleted && (len(commentsTree.Replies) == 0 || !rd.visible) {
 			continue
 		}
 
-		commentsTree.tsModified, commentsTree.tsCreated = tsModified, tsCreated
-		if commentsTree.tsCreated.Before(res.Info.FirstTS) {
-			res.Info.FirstTS = commentsTree.tsCreated
-		}
-		if commentsTree.tsModified.After(res.Info.LastTS) {
-			res.Info.LastTS = commentsTree.tsModified
-		}
-
-		res.Info.ReadOnly = readOnlyAge > 0 && !res.Info.FirstTS.IsZero() &&
-			res.Info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-
 		res.Nodes = append(res.Nodes, commentsTree)
 	}
 
@@ -79,7 +57,7 @@ func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree
 }
 
 // proc makes tree for one top-level comment recursively
-func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (result *Node, modified, created time.Time) {
+func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (result *Node) {
 	if rd.tsModified.IsZero() || rd.tsCreated.IsZero() {
 		rd.tsModified, rd.tsCreated = node.Comment.Timestamp, node.Comment.Timestamp
 	}
@@ -106,7 +84,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 	sort.Slice(node.Replies, func(i, j int) bool {
 		return node.Replies[i].Comment.Timestamp.Before(node.Replies[j].Comment.Timestamp)
 	})
-	return node, rd.tsModified, rd.tsCreated
+	node.tsModified, node.tsCreated = rd.tsModified, rd.tsCreated
+	return node
 }
 
 // filter returns comments for parentID

backend/app/store/service/tree_test.go

@@ -37,19 +37,15 @@ func TestMakeTree(t *testing.T) {
 		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
 	}
 
-	res := MakeTree(comments, "time", 0)
+	res := MakeTree(comments, "time")
 	resJSON, err := json.Marshal(&res)
 	require.NoError(t, err)
 
 	expJSON := mustLoadJSONFile(t, "testdata/tree.json")
 	assert.Equal(t, expJSON, resJSON)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
 
-	res = MakeTree([]store.Comment{}, "time", 0)
+	res = MakeTree([]store.Comment{}, "time")
 	assert.Equal(t, &Tree{}, res)
-
-	res = MakeTree(comments, "time", 10)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
 }
 
 func TestMakeEmptySubtree(t *testing.T) {
@@ -79,7 +75,7 @@ func TestMakeEmptySubtree(t *testing.T) {
 		{Locator: loc, ID: "3", Timestamp: ts(48, 1), Deleted: true}, // deleted top level
 	}
 
-	res := MakeTree(comments, "time", 0)
+	res := MakeTree(comments, "time")
 	resJSON, err := json.Marshal(&res)
 	require.NoError(t, err)
 	t.Log(string(resJSON))
@@ -108,50 +104,50 @@ func TestTreeSortNodes(t *testing.T) {
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
 	}
 
-	res := MakeTree(comments, "+active", 0)
+	res := MakeTree(comments, "+active")
 	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 
-	res = MakeTree(comments, "-active", 0)
+	res = MakeTree(comments, "-active")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "+time", 0)
+	res = MakeTree(comments, "+time")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "-time", 0)
+	res = MakeTree(comments, "-time")
 	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "score", 0)
+	res = MakeTree(comments, "score")
 	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "+score", 0)
+	res = MakeTree(comments, "+score")
 	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "-score", 0)
+	res = MakeTree(comments, "-score")
 	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "+controversy", 0)
+	res = MakeTree(comments, "+controversy")
 	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
 
-	res = MakeTree(comments, "-controversy", 0)
+	res = MakeTree(comments, "-controversy")
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "undefined", 0)
+	res = MakeTree(comments, "undefined")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 }
@@ -164,7 +160,7 @@ func BenchmarkTree(b *testing.B) {
 	assert.NoError(b, err)
 
 	for i := 0; i < b.N; i++ {
-		res := MakeTree(comments, "time", 0)
+		res := MakeTree(comments, "time")
 		assert.NotNil(b, res)
 	}
 }

backend/go.mod

@@ -1,78 +1,74 @@
 module github.com/umputun/remark42/backend
 
-go 1.17
+go 1.21
 
 require (
 	github.com/Depado/bfchroma/v2 v2.0.0
-	github.com/PuerkitoBio/goquery v1.8.0
-	github.com/alecthomas/chroma/v2 v2.3.0
-	github.com/didip/tollbooth/v7 v7.0.1
+	github.com/PuerkitoBio/goquery v1.9.2
+	github.com/alecthomas/chroma/v2 v2.13.0
+	github.com/didip/tollbooth/v7 v7.0.2
 	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f
-	github.com/go-chi/chi/v5 v5.0.7
+	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1
-	github.com/go-chi/render v1.0.2
-	github.com/go-pkgz/auth v1.20.0
+	github.com/go-chi/render v1.0.3
+	github.com/go-pkgz/auth v1.24.2
 	github.com/go-pkgz/jrpc v0.3.0
-	github.com/go-pkgz/lcw v1.0.1
-	github.com/go-pkgz/lgr v0.10.4
-	github.com/go-pkgz/notify v0.2.0
+	github.com/go-pkgz/lcw/v2 v2.0.0
+	github.com/go-pkgz/lgr v0.11.1
+	github.com/go-pkgz/notify v1.1.1
 	github.com/go-pkgz/repeater v1.1.3
-	github.com/go-pkgz/rest v1.16.0
-	github.com/go-pkgz/syncs v1.2.0
+	github.com/go-pkgz/rest v1.19.0
+	github.com/go-pkgz/syncs v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
-	github.com/google/uuid v1.3.0
-	github.com/gorilla/feeds v1.1.1
+	github.com/google/uuid v1.6.0
+	github.com/gorilla/feeds v1.2.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/jessevdk/go-flags v1.5.0
-	github.com/kyokomi/emoji/v2 v2.2.10
-	github.com/microcosm-cc/bluemonday v1.0.20
-	github.com/rs/xid v1.4.0
+	github.com/jessevdk/go-flags v1.6.1
+	github.com/kyokomi/emoji/v2 v2.2.13
+	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/rs/xid v1.5.0
 	github.com/russross/blackfriday/v2 v2.1.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
-	github.com/stretchr/testify v1.8.0
-	go.etcd.io/bbolt v1.3.6
-	go.uber.org/goleak v1.2.0
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
-	golang.org/x/image v0.0.0-20220617043117-41969df76e82
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
+	github.com/stretchr/testify v1.9.0
+	go.etcd.io/bbolt v1.3.10
+	go.uber.org/goleak v1.3.0
+	golang.org/x/crypto v0.25.0
+	golang.org/x/image v0.18.0
+	golang.org/x/net v0.27.0
 )
 
 require (
-	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	github.com/ajg/form v1.5.1 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dghubble/oauth1 v0.7.1 // indirect
+	github.com/dghubble/oauth1 v0.7.3 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/go-oauth2/oauth2/v4 v4.5.1 // indirect
-	github.com/go-pkgz/email v0.4.0 // indirect
-	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/go-oauth2/oauth2/v4 v4.5.2 // indirect
+	github.com/go-pkgz/email v0.5.0 // indirect
+	github.com/go-pkgz/expirable-cache/v3 v3.0.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/klauspost/compress v1.15.2 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
-	github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/slack-go/slack v0.11.3 // indirect
+	github.com/redis/go-redis/v9 v9.4.0 // indirect
+	github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d // indirect
+	github.com/slack-go/slack v0.12.5 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.1 // indirect
-	github.com/xdg-go/stringprep v1.0.3 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.mongodb.org/mongo-driver v1.10.2 // indirect
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	go.mongodb.org/mongo-driver v1.16.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

backend/remark.rest

@@ -103,6 +103,14 @@ GET {{host}}/api/v1/admin/blocked?site={{site}}
 ### delete comment by id
 DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
 
+### export site (for backup)
+GET {{host}}/api/v1/admin/export?site={{site}}&mode=stream
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+
+### export site (for backup) to .gz file
+GET {{host}}/api/v1/admin/export?site={{site}}&mode=file
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+
 ### get post info
 GET {{host}}/api/v1/info?site={{site}}&url={{url}}
 
@@ -119,12 +127,27 @@ GET {{host}}/api/v1/rss/reply?site={{site}}&user={{user}}
 GET {{host}}/api/v1/avatar/blah
 
 ### send confirmation token for current user to specified email. auth token for dev user for secret=12345.
-POST {{host}}/api/v1/email/subscribe?site={{site}}&address={{email}}
+### in case the user logged in with the same email, it will be confirmed right away with "updated" set to "true" in the response,
+### and no email will be sent.
+POST {{host}}/api/v1/email/subscribe
 X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+Content-Type: application/json
+
+{
+	"site": "{{site}}",
+	"address": "{{email}}"
+}
+
 
 ### add email for notifications for current user via token from email. auth token for dev user for secret=12345.
-POST {{host}}/api/v1/email/confirm?site={{site}}&tkn={{token}}
+POST {{host}}/api/v1/email/confirm
 X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+Content-Type: application/json
+
+{
+	"site": "{{site}}",
+	"token": "{{token}}"
+}
 
 ### get current user email. auth token for dev user for secret=12345.
 GET {{host}}/api/v1/email?site={{site}}

backend/vendor/cloud.google.com/go/compute/metadata/CHANGES.md

@@ -0,0 +1,45 @@
+# Changes
+
+## [0.5.0](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.4.0...compute/metadata/v0.5.0) (2024-07-10)
+
+
+### Features
+
+* **compute/metadata:** Add sys check for windows OnGCE ([#10521](https://github.com/googleapis/google-cloud-go/issues/10521)) ([3b9a830](https://github.com/googleapis/google-cloud-go/commit/3b9a83063960d2a2ac20beb47cc15818a68bd302))
+
+## [0.4.0](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.3.0...compute/metadata/v0.4.0) (2024-07-01)
+
+
+### Features
+
+* **compute/metadata:** Add context for all functions/methods ([#10370](https://github.com/googleapis/google-cloud-go/issues/10370)) ([66b8efe](https://github.com/googleapis/google-cloud-go/commit/66b8efe7ad877e052b2987bb4475477e38c67bb3))
+
+
+### Documentation
+
+* **compute/metadata:** Update OnGCE description ([#10408](https://github.com/googleapis/google-cloud-go/issues/10408)) ([6a46dca](https://github.com/googleapis/google-cloud-go/commit/6a46dca4eae4f88ec6f88822e01e5bf8aeca787f))
+
+## [0.3.0](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.3...compute/metadata/v0.3.0) (2024-04-15)
+
+
+### Features
+
+* **compute/metadata:** Add context aware functions  ([#9733](https://github.com/googleapis/google-cloud-go/issues/9733)) ([e4eb5b4](https://github.com/googleapis/google-cloud-go/commit/e4eb5b46ee2aec9d2fc18300bfd66015e25a0510))
+
+## [0.2.3](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.2...compute/metadata/v0.2.3) (2022-12-15)
+
+
+### Bug Fixes
+
+* **compute/metadata:** Switch DNS lookup to an absolute lookup ([119b410](https://github.com/googleapis/google-cloud-go/commit/119b41060c7895e45e48aee5621ad35607c4d021)), refs [#7165](https://github.com/googleapis/google-cloud-go/issues/7165)
+
+## [0.2.2](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.1...compute/metadata/v0.2.2) (2022-12-01)
+
+
+### Bug Fixes
+
+* **compute/metadata:** Set IdleConnTimeout for http.Client ([#7084](https://github.com/googleapis/google-cloud-go/issues/7084)) ([766516a](https://github.com/googleapis/google-cloud-go/commit/766516aaf3816bfb3159efeea65aa3d1d205a3e2)), refs [#5430](https://github.com/googleapis/google-cloud-go/issues/5430)
+
+## [0.1.0] (2022-10-26)
+
+Initial release of metadata being it's own module.

backend/vendor/cloud.google.com/go/compute/metadata/README.md

@@ -0,0 +1,27 @@
+# Compute API
+
+[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata)
+
+This is a utility library for communicating with Google Cloud metadata service
+on Google Cloud.
+
+## Install
+
+```bash
+go get cloud.google.com/go/compute/metadata
+```
+
+## Go Version Support
+
+See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported)
+section in the root directory's README.
+
+## Contributing
+
+Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md)
+document for details.
+
+Please note that this project is released with a Contributor Code of Conduct.
+By participating in this project you agree to abide by its terms. See
+[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct)
+for more information.

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -16,19 +16,18 @@
 // metadata and API service accounts.
 //
 // This package is a wrapper around the GCE metadata service,
-// as documented at https://developers.google.com/compute/docs/metadata.
+// as documented at https://cloud.google.com/compute/docs/metadata/overview.
 package metadata // import "cloud.google.com/go/compute/metadata"
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -70,7 +69,9 @@ func newDefaultHTTPClient() *http.Client {
 				Timeout:   2 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).Dial,
+			IdleConnTimeout: 60 * time.Second,
 		},
+		Timeout: 5 * time.Second,
 	}
 }
 
@@ -86,16 +87,16 @@ func (suffix NotDefinedError) Error() string {
 	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
 }
 
-func (c *cachedValue) get(cl *Client) (v string, err error) {
+func (c *cachedValue) get(ctx context.Context, cl *Client) (v string, err error) {
 	defer c.mu.Unlock()
 	c.mu.Lock()
 	if c.v != "" {
 		return c.v, nil
 	}
 	if c.trim {
-		v, err = cl.getTrimmed(c.k)
+		v, err = cl.getTrimmed(ctx, c.k)
 	} else {
-		v, err = cl.Get(c.k)
+		v, err = cl.GetWithContext(ctx, c.k)
 	}
 	if err == nil {
 		c.v = v
@@ -108,7 +109,9 @@ var (
 	onGCE     bool
 )
 
-// OnGCE reports whether this process is running on Google Compute Engine.
+// OnGCE reports whether this process is running on Google Compute Platforms.
+// NOTE: True returned from `OnGCE` does not guarantee that the metadata server
+// is accessible from this process and have all the metadata defined.
 func OnGCE() bool {
 	onGCEOnce.Do(initOnGCE)
 	return onGCE
@@ -145,7 +148,7 @@ func testOnGCE() bool {
 
 	go func() {
 		resolver := &net.Resolver{}
-		addrs, err := resolver.LookupHost(ctx, "metadata.google.internal")
+		addrs, err := resolver.LookupHost(ctx, "metadata.google.internal.")
 		if err != nil || len(addrs) == 0 {
 			resc <- false
 			return
@@ -186,78 +189,213 @@ func testOnGCE() bool {
 	return <-resc
 }
 
-// systemInfoSuggestsGCE reports whether the local system (without
-// doing network requests) suggests that we're running on GCE. If this
-// returns true, testOnGCE tries a bit harder to reach its metadata
-// server.
-func systemInfoSuggestsGCE() bool {
-	if runtime.GOOS != "linux" {
-		// We don't have any non-Linux clues available, at least yet.
-		return false
-	}
-	slurp, _ := ioutil.ReadFile("/sys/class/dmi/id/product_name")
-	name := strings.TrimSpace(string(slurp))
-	return name == "Google" || name == "Google Compute Engine"
-}
-
-// Subscribe calls Client.Subscribe on the default client.
+// Subscribe calls Client.SubscribeWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [SubscribeWithContext].
 func Subscribe(suffix string, fn func(v string, ok bool) error) error {
-	return defaultClient.Subscribe(suffix, fn)
+	return defaultClient.SubscribeWithContext(context.Background(), suffix, func(ctx context.Context, v string, ok bool) error { return fn(v, ok) })
 }
 
-// Get calls Client.Get on the default client.
-func Get(suffix string) (string, error) { return defaultClient.Get(suffix) }
+// SubscribeWithContext calls Client.SubscribeWithContext on the default client.
+func SubscribeWithContext(ctx context.Context, suffix string, fn func(ctx context.Context, v string, ok bool) error) error {
+	return defaultClient.SubscribeWithContext(ctx, suffix, fn)
+}
+
+// Get calls Client.GetWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [GetWithContext].
+func Get(suffix string) (string, error) {
+	return defaultClient.GetWithContext(context.Background(), suffix)
+}
+
+// GetWithContext calls Client.GetWithContext on the default client.
+func GetWithContext(ctx context.Context, suffix string) (string, error) {
+	return defaultClient.GetWithContext(ctx, suffix)
+}
 
 // ProjectID returns the current instance's project ID string.
-func ProjectID() (string, error) { return defaultClient.ProjectID() }
+//
+// Deprecated: Please use the context aware variant [ProjectIDWithContext].
+func ProjectID() (string, error) {
+	return defaultClient.ProjectIDWithContext(context.Background())
+}
+
+// ProjectIDWithContext returns the current instance's project ID string.
+func ProjectIDWithContext(ctx context.Context) (string, error) {
+	return defaultClient.ProjectIDWithContext(ctx)
+}
 
 // NumericProjectID returns the current instance's numeric project ID.
-func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() }
+//
+// Deprecated: Please use the context aware variant [NumericProjectIDWithContext].
+func NumericProjectID() (string, error) {
+	return defaultClient.NumericProjectIDWithContext(context.Background())
+}
+
+// NumericProjectIDWithContext returns the current instance's numeric project ID.
+func NumericProjectIDWithContext(ctx context.Context) (string, error) {
+	return defaultClient.NumericProjectIDWithContext(ctx)
+}
 
 // InternalIP returns the instance's primary internal IP address.
-func InternalIP() (string, error) { return defaultClient.InternalIP() }
+//
+// Deprecated: Please use the context aware variant [InternalIPWithContext].
+func InternalIP() (string, error) {
+	return defaultClient.InternalIPWithContext(context.Background())
+}
+
+// InternalIPWithContext returns the instance's primary internal IP address.
+func InternalIPWithContext(ctx context.Context) (string, error) {
+	return defaultClient.InternalIPWithContext(ctx)
+}
 
 // ExternalIP returns the instance's primary external (public) IP address.
-func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
+//
+// Deprecated: Please use the context aware variant [ExternalIPWithContext].
+func ExternalIP() (string, error) {
+	return defaultClient.ExternalIPWithContext(context.Background())
+}
 
-// Email calls Client.Email on the default client.
-func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) }
+// ExternalIPWithContext returns the instance's primary external (public) IP address.
+func ExternalIPWithContext(ctx context.Context) (string, error) {
+	return defaultClient.ExternalIPWithContext(ctx)
+}
+
+// Email calls Client.EmailWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [EmailWithContext].
+func Email(serviceAccount string) (string, error) {
+	return defaultClient.EmailWithContext(context.Background(), serviceAccount)
+}
+
+// EmailWithContext calls Client.EmailWithContext on the default client.
+func EmailWithContext(ctx context.Context, serviceAccount string) (string, error) {
+	return defaultClient.EmailWithContext(ctx, serviceAccount)
+}
 
 // Hostname returns the instance's hostname. This will be of the form
 // "<instanceID>.c.<projID>.internal".
-func Hostname() (string, error) { return defaultClient.Hostname() }
+//
+// Deprecated: Please use the context aware variant [HostnameWithContext].
+func Hostname() (string, error) {
+	return defaultClient.HostnameWithContext(context.Background())
+}
+
+// HostnameWithContext returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func HostnameWithContext(ctx context.Context) (string, error) {
+	return defaultClient.HostnameWithContext(ctx)
+}
 
 // InstanceTags returns the list of user-defined instance tags,
 // assigned when initially creating a GCE instance.
-func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() }
+//
+// Deprecated: Please use the context aware variant [InstanceTagsWithContext].
+func InstanceTags() ([]string, error) {
+	return defaultClient.InstanceTagsWithContext(context.Background())
+}
+
+// InstanceTagsWithContext returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTagsWithContext(ctx context.Context) ([]string, error) {
+	return defaultClient.InstanceTagsWithContext(ctx)
+}
 
 // InstanceID returns the current VM's numeric instance ID.
-func InstanceID() (string, error) { return defaultClient.InstanceID() }
+//
+// Deprecated: Please use the context aware variant [InstanceIDWithContext].
+func InstanceID() (string, error) {
+	return defaultClient.InstanceIDWithContext(context.Background())
+}
+
+// InstanceIDWithContext returns the current VM's numeric instance ID.
+func InstanceIDWithContext(ctx context.Context) (string, error) {
+	return defaultClient.InstanceIDWithContext(ctx)
+}
 
 // InstanceName returns the current VM's instance ID string.
-func InstanceName() (string, error) { return defaultClient.InstanceName() }
+//
+// Deprecated: Please use the context aware variant [InstanceNameWithContext].
+func InstanceName() (string, error) {
+	return defaultClient.InstanceNameWithContext(context.Background())
+}
+
+// InstanceNameWithContext returns the current VM's instance ID string.
+func InstanceNameWithContext(ctx context.Context) (string, error) {
+	return defaultClient.InstanceNameWithContext(ctx)
+}
 
 // Zone returns the current VM's zone, such as "us-central1-b".
-func Zone() (string, error) { return defaultClient.Zone() }
+//
+// Deprecated: Please use the context aware variant [ZoneWithContext].
+func Zone() (string, error) {
+	return defaultClient.ZoneWithContext(context.Background())
+}
 
-// InstanceAttributes calls Client.InstanceAttributes on the default client.
-func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() }
+// ZoneWithContext returns the current VM's zone, such as "us-central1-b".
+func ZoneWithContext(ctx context.Context) (string, error) {
+	return defaultClient.ZoneWithContext(ctx)
+}
 
-// ProjectAttributes calls Client.ProjectAttributes on the default client.
-func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() }
+// InstanceAttributes calls Client.InstanceAttributesWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [InstanceAttributesWithContext.
+func InstanceAttributes() ([]string, error) {
+	return defaultClient.InstanceAttributesWithContext(context.Background())
+}
 
-// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client.
+// InstanceAttributesWithContext calls Client.ProjectAttributesWithContext on the default client.
+func InstanceAttributesWithContext(ctx context.Context) ([]string, error) {
+	return defaultClient.InstanceAttributesWithContext(ctx)
+}
+
+// ProjectAttributes calls Client.ProjectAttributesWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [ProjectAttributesWithContext].
+func ProjectAttributes() ([]string, error) {
+	return defaultClient.ProjectAttributesWithContext(context.Background())
+}
+
+// ProjectAttributesWithContext calls Client.ProjectAttributesWithContext on the default client.
+func ProjectAttributesWithContext(ctx context.Context) ([]string, error) {
+	return defaultClient.ProjectAttributesWithContext(ctx)
+}
+
+// InstanceAttributeValue calls Client.InstanceAttributeValueWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [InstanceAttributeValueWithContext].
 func InstanceAttributeValue(attr string) (string, error) {
-	return defaultClient.InstanceAttributeValue(attr)
+	return defaultClient.InstanceAttributeValueWithContext(context.Background(), attr)
 }
 
-// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client.
+// InstanceAttributeValueWithContext calls Client.InstanceAttributeValueWithContext on the default client.
+func InstanceAttributeValueWithContext(ctx context.Context, attr string) (string, error) {
+	return defaultClient.InstanceAttributeValueWithContext(ctx, attr)
+}
+
+// ProjectAttributeValue calls Client.ProjectAttributeValueWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [ProjectAttributeValueWithContext].
 func ProjectAttributeValue(attr string) (string, error) {
-	return defaultClient.ProjectAttributeValue(attr)
+	return defaultClient.ProjectAttributeValueWithContext(context.Background(), attr)
 }
 
-// Scopes calls Client.Scopes on the default client.
-func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) }
+// ProjectAttributeValueWithContext calls Client.ProjectAttributeValueWithContext on the default client.
+func ProjectAttributeValueWithContext(ctx context.Context, attr string) (string, error) {
+	return defaultClient.ProjectAttributeValueWithContext(ctx, attr)
+}
+
+// Scopes calls Client.ScopesWithContext on the default client.
+//
+// Deprecated: Please use the context aware variant [ScopesWithContext].
+func Scopes(serviceAccount string) ([]string, error) {
+	return defaultClient.ScopesWithContext(context.Background(), serviceAccount)
+}
+
+// ScopesWithContext calls Client.ScopesWithContext on the default client.
+func ScopesWithContext(ctx context.Context, serviceAccount string) ([]string, error) {
+	return defaultClient.ScopesWithContext(ctx, serviceAccount)
+}
 
 func strsContains(ss []string, s string) bool {
 	for _, v := range ss {
@@ -280,14 +418,12 @@ func NewClient(c *http.Client) *Client {
 	if c == nil {
 		return defaultClient
 	}
-
 	return &Client{hc: c}
 }
 
 // getETag returns a value from the metadata service as well as the associated ETag.
 // This func is otherwise equivalent to Get.
-func (c *Client) getETag(suffix string) (value, etag string, err error) {
-	ctx := context.TODO()
+func (c *Client) getETag(ctx context.Context, suffix string) (value, etag string, err error) {
 	// Using a fixed IP makes it very difficult to spoof the metadata service in
 	// a container, which is an important use-case for local testing of cloud
 	// deployments. To enable spoofing of the metadata service, the environment
@@ -304,7 +440,7 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 	}
 	suffix = strings.TrimLeft(suffix, "/")
 	u := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, err := http.NewRequest("GET", u, nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
 	if err != nil {
 		return "", "", err
 	}
@@ -334,7 +470,7 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 	if res.StatusCode == http.StatusNotFound {
 		return "", "", NotDefinedError(suffix)
 	}
-	all, err := ioutil.ReadAll(res.Body)
+	all, err := io.ReadAll(res.Body)
 	if err != nil {
 		return "", "", err
 	}
@@ -352,19 +488,37 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 //
 // If the requested metadata is not defined, the returned error will
 // be of type NotDefinedError.
+//
+// Deprecated: Please use the context aware variant [Client.GetWithContext].
 func (c *Client) Get(suffix string) (string, error) {
-	val, _, err := c.getETag(suffix)
+	return c.GetWithContext(context.Background(), suffix)
+}
+
+// GetWithContext returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// NOTE: Without an extra deadline in the context this call can take in the
+// worst case, with internal backoff retries, up to 15 seconds (e.g. when server
+// is responding slowly). Pass context with additional timeouts when needed.
+func (c *Client) GetWithContext(ctx context.Context, suffix string) (string, error) {
+	val, _, err := c.getETag(ctx, suffix)
 	return val, err
 }
 
-func (c *Client) getTrimmed(suffix string) (s string, err error) {
-	s, err = c.Get(suffix)
+func (c *Client) getTrimmed(ctx context.Context, suffix string) (s string, err error) {
+	s, err = c.GetWithContext(ctx, suffix)
 	s = strings.TrimSpace(s)
 	return
 }
 
-func (c *Client) lines(suffix string) ([]string, error) {
-	j, err := c.Get(suffix)
+func (c *Client) lines(ctx context.Context, suffix string) ([]string, error) {
+	j, err := c.GetWithContext(ctx, suffix)
 	if err != nil {
 		return nil, err
 	}
@@ -376,45 +530,104 @@ func (c *Client) lines(suffix string) ([]string, error) {
 }
 
 // ProjectID returns the current instance's project ID string.
-func (c *Client) ProjectID() (string, error) { return projID.get(c) }
+//
+// Deprecated: Please use the context aware variant [Client.ProjectIDWithContext].
+func (c *Client) ProjectID() (string, error) { return c.ProjectIDWithContext(context.Background()) }
+
+// ProjectIDWithContext returns the current instance's project ID string.
+func (c *Client) ProjectIDWithContext(ctx context.Context) (string, error) { return projID.get(ctx, c) }
 
 // NumericProjectID returns the current instance's numeric project ID.
-func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) }
+//
+// Deprecated: Please use the context aware variant [Client.NumericProjectIDWithContext].
+func (c *Client) NumericProjectID() (string, error) {
+	return c.NumericProjectIDWithContext(context.Background())
+}
+
+// NumericProjectIDWithContext returns the current instance's numeric project ID.
+func (c *Client) NumericProjectIDWithContext(ctx context.Context) (string, error) {
+	return projNum.get(ctx, c)
+}
 
 // InstanceID returns the current VM's numeric instance ID.
-func (c *Client) InstanceID() (string, error) { return instID.get(c) }
+//
+// Deprecated: Please use the context aware variant [Client.InstanceIDWithContext].
+func (c *Client) InstanceID() (string, error) {
+	return c.InstanceIDWithContext(context.Background())
+}
+
+// InstanceIDWithContext returns the current VM's numeric instance ID.
+func (c *Client) InstanceIDWithContext(ctx context.Context) (string, error) {
+	return instID.get(ctx, c)
+}
 
 // InternalIP returns the instance's primary internal IP address.
+//
+// Deprecated: Please use the context aware variant [Client.InternalIPWithContext].
 func (c *Client) InternalIP() (string, error) {
-	return c.getTrimmed("instance/network-interfaces/0/ip")
+	return c.InternalIPWithContext(context.Background())
+}
+
+// InternalIPWithContext returns the instance's primary internal IP address.
+func (c *Client) InternalIPWithContext(ctx context.Context) (string, error) {
+	return c.getTrimmed(ctx, "instance/network-interfaces/0/ip")
 }
 
 // Email returns the email address associated with the service account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
+//
+// Deprecated: Please use the context aware variant [Client.EmailWithContext].
 func (c *Client) Email(serviceAccount string) (string, error) {
+	return c.EmailWithContext(context.Background(), serviceAccount)
+}
+
+// EmailWithContext returns the email address associated with the service account.
+// The serviceAccount parameter default value (empty string or "default" value)
+// will use the instance's main account.
+func (c *Client) EmailWithContext(ctx context.Context, serviceAccount string) (string, error) {
 	if serviceAccount == "" {
 		serviceAccount = "default"
 	}
-	return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email")
+	return c.getTrimmed(ctx, "instance/service-accounts/"+serviceAccount+"/email")
 }
 
 // ExternalIP returns the instance's primary external (public) IP address.
+//
+// Deprecated: Please use the context aware variant [Client.ExternalIPWithContext].
 func (c *Client) ExternalIP() (string, error) {
-	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+	return c.ExternalIPWithContext(context.Background())
+}
+
+// ExternalIPWithContext returns the instance's primary external (public) IP address.
+func (c *Client) ExternalIPWithContext(ctx context.Context) (string, error) {
+	return c.getTrimmed(ctx, "instance/network-interfaces/0/access-configs/0/external-ip")
 }
 
 // Hostname returns the instance's hostname. This will be of the form
 // "<instanceID>.c.<projID>.internal".
+//
+// Deprecated: Please use the context aware variant [Client.HostnameWithContext].
 func (c *Client) Hostname() (string, error) {
-	return c.getTrimmed("instance/hostname")
+	return c.HostnameWithContext(context.Background())
 }
 
-// InstanceTags returns the list of user-defined instance tags,
-// assigned when initially creating a GCE instance.
+// HostnameWithContext returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func (c *Client) HostnameWithContext(ctx context.Context) (string, error) {
+	return c.getTrimmed(ctx, "instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags.
+//
+// Deprecated: Please use the context aware variant [Client.InstanceTagsWithContext].
 func (c *Client) InstanceTags() ([]string, error) {
+	return c.InstanceTagsWithContext(context.Background())
+}
+
+// InstanceTagsWithContext returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func (c *Client) InstanceTagsWithContext(ctx context.Context) ([]string, error) {
 	var s []string
-	j, err := c.Get("instance/tags")
+	j, err := c.GetWithContext(ctx, "instance/tags")
 	if err != nil {
 		return nil, err
 	}
@@ -425,13 +638,27 @@ func (c *Client) InstanceTags() ([]string, error) {
 }
 
 // InstanceName returns the current VM's instance ID string.
+//
+// Deprecated: Please use the context aware variant [Client.InstanceNameWithContext].
 func (c *Client) InstanceName() (string, error) {
-	return c.getTrimmed("instance/name")
+	return c.InstanceNameWithContext(context.Background())
+}
+
+// InstanceNameWithContext returns the current VM's instance ID string.
+func (c *Client) InstanceNameWithContext(ctx context.Context) (string, error) {
+	return c.getTrimmed(ctx, "instance/name")
 }
 
 // Zone returns the current VM's zone, such as "us-central1-b".
+//
+// Deprecated: Please use the context aware variant [Client.ZoneWithContext].
 func (c *Client) Zone() (string, error) {
-	zone, err := c.getTrimmed("instance/zone")
+	return c.ZoneWithContext(context.Background())
+}
+
+// ZoneWithContext returns the current VM's zone, such as "us-central1-b".
+func (c *Client) ZoneWithContext(ctx context.Context) (string, error) {
+	zone, err := c.getTrimmed(ctx, "instance/zone")
 	// zone is of the form "projects/<projNum>/zones/<zoneName>".
 	if err != nil {
 		return "", err
@@ -442,12 +669,34 @@ func (c *Client) Zone() (string, error) {
 // InstanceAttributes returns the list of user-defined attributes,
 // assigned when initially creating a GCE VM instance. The value of an
 // attribute can be obtained with InstanceAttributeValue.
-func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") }
+//
+// Deprecated: Please use the context aware variant [Client.InstanceAttributesWithContext].
+func (c *Client) InstanceAttributes() ([]string, error) {
+	return c.InstanceAttributesWithContext(context.Background())
+}
+
+// InstanceAttributesWithContext returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func (c *Client) InstanceAttributesWithContext(ctx context.Context) ([]string, error) {
+	return c.lines(ctx, "instance/attributes/")
+}
 
 // ProjectAttributes returns the list of user-defined attributes
 // applying to the project as a whole, not just this VM.  The value of
 // an attribute can be obtained with ProjectAttributeValue.
-func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") }
+//
+// Deprecated: Please use the context aware variant [Client.ProjectAttributesWithContext].
+func (c *Client) ProjectAttributes() ([]string, error) {
+	return c.ProjectAttributesWithContext(context.Background())
+}
+
+// ProjectAttributesWithContext returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func (c *Client) ProjectAttributesWithContext(ctx context.Context) ([]string, error) {
+	return c.lines(ctx, "project/attributes/")
+}
 
 // InstanceAttributeValue returns the value of the provided VM
 // instance attribute.
@@ -457,8 +706,22 @@ func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project
 //
 // InstanceAttributeValue may return ("", nil) if the attribute was
 // defined to be the empty string.
+//
+// Deprecated: Please use the context aware variant [Client.InstanceAttributeValueWithContext].
 func (c *Client) InstanceAttributeValue(attr string) (string, error) {
-	return c.Get("instance/attributes/" + attr)
+	return c.InstanceAttributeValueWithContext(context.Background(), attr)
+}
+
+// InstanceAttributeValueWithContext returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) InstanceAttributeValueWithContext(ctx context.Context, attr string) (string, error) {
+	return c.GetWithContext(ctx, "instance/attributes/"+attr)
 }
 
 // ProjectAttributeValue returns the value of the provided
@@ -469,39 +732,71 @@ func (c *Client) InstanceAttributeValue(attr string) (string, error) {
 //
 // ProjectAttributeValue may return ("", nil) if the attribute was
 // defined to be the empty string.
+//
+// Deprecated: Please use the context aware variant [Client.ProjectAttributeValueWithContext].
 func (c *Client) ProjectAttributeValue(attr string) (string, error) {
-	return c.Get("project/attributes/" + attr)
+	return c.ProjectAttributeValueWithContext(context.Background(), attr)
+}
+
+// ProjectAttributeValueWithContext returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) ProjectAttributeValueWithContext(ctx context.Context, attr string) (string, error) {
+	return c.GetWithContext(ctx, "project/attributes/"+attr)
 }
 
 // Scopes returns the service account scopes for the given account.
 // The account may be empty or the string "default" to use the instance's
 // main account.
+//
+// Deprecated: Please use the context aware variant [Client.ScopesWithContext].
 func (c *Client) Scopes(serviceAccount string) ([]string, error) {
+	return c.ScopesWithContext(context.Background(), serviceAccount)
+}
+
+// ScopesWithContext returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) ScopesWithContext(ctx context.Context, serviceAccount string) ([]string, error) {
 	if serviceAccount == "" {
 		serviceAccount = "default"
 	}
-	return c.lines("instance/service-accounts/" + serviceAccount + "/scopes")
+	return c.lines(ctx, "instance/service-accounts/"+serviceAccount+"/scopes")
 }
 
 // Subscribe subscribes to a value from the metadata service.
 // The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
 // The suffix may contain query parameters.
 //
-// Subscribe calls fn with the latest metadata value indicated by the provided
-// suffix. If the metadata value is deleted, fn is called with the empty string
-// and ok false. Subscribe blocks until fn returns a non-nil error or the value
-// is deleted. Subscribe returns the error value returned from the last call to
-// fn, which may be nil when ok == false.
+// Deprecated: Please use the context aware variant [Client.SubscribeWithContext].
 func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	return c.SubscribeWithContext(context.Background(), suffix, func(ctx context.Context, v string, ok bool) error { return fn(v, ok) })
+}
+
+// SubscribeWithContext subscribes to a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+// The suffix may contain query parameters.
+//
+// SubscribeWithContext calls fn with the latest metadata value indicated by the
+// provided suffix. If the metadata value is deleted, fn is called with the
+// empty string and ok false. Subscribe blocks until fn returns a non-nil error
+// or the value is deleted. Subscribe returns the error value returned from the
+// last call to fn, which may be nil when ok == false.
+func (c *Client) SubscribeWithContext(ctx context.Context, suffix string, fn func(ctx context.Context, v string, ok bool) error) error {
 	const failedSubscribeSleep = time.Second * 5
 
 	// First check to see if the metadata value exists at all.
-	val, lastETag, err := c.getETag(suffix)
+	val, lastETag, err := c.getETag(ctx, suffix)
 	if err != nil {
 		return err
 	}
 
-	if err := fn(val, true); err != nil {
+	if err := fn(ctx, val, true); err != nil {
 		return err
 	}
 
@@ -512,7 +807,7 @@ func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) erro
 		suffix += "?wait_for_change=true&last_etag="
 	}
 	for {
-		val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag))
+		val, etag, err := c.getETag(ctx, suffix+url.QueryEscape(lastETag))
 		if err != nil {
 			if _, deleted := err.(NotDefinedError); !deleted {
 				time.Sleep(failedSubscribeSleep)
@@ -522,7 +817,7 @@ func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) erro
 		}
 		lastETag = etag
 
-		if err := fn(val, ok); err != nil || !ok {
+		if err := fn(ctx, val, ok); err != nil || !ok {
 			return err
 		}
 	}

backend/vendor/cloud.google.com/go/compute/metadata/retry.go

@@ -27,7 +27,7 @@ const (
 )
 
 var (
-	syscallRetryable = func(err error) bool { return false }
+	syscallRetryable = func(error) bool { return false }
 )
 
 // defaultBackoff is basically equivalent to gax.Backoff without the need for

backend/vendor/cloud.google.com/go/compute/metadata/syscheck.go

@@ -0,0 +1,26 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !windows && !linux
+
+package metadata
+
+// systemInfoSuggestsGCE reports whether the local system (without
+// doing network requests) suggests that we're running on GCE. If this
+// returns true, testOnGCE tries a bit harder to reach its metadata
+// server.
+func systemInfoSuggestsGCE() bool {
+	// We don't currently have checks for other GOOS
+	return false
+}

backend/vendor/cloud.google.com/go/compute/metadata/syscheck_linux.go

@@ -0,0 +1,28 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+
+package metadata
+
+import (
+	"os"
+	"strings"
+)
+
+func systemInfoSuggestsGCE() bool {
+	b, _ := os.ReadFile("/sys/class/dmi/id/product_name")
+	name := strings.TrimSpace(string(b))
+	return name == "Google" || name == "Google Compute Engine"
+}

backend/vendor/cloud.google.com/go/compute/metadata/syscheck_windows.go

@@ -0,0 +1,38 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package metadata
+
+import (
+	"strings"
+
+	"golang.org/x/sys/windows/registry"
+)
+
+func systemInfoSuggestsGCE() bool {
+	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\HardwareConfig\Current`, registry.QUERY_VALUE)
+	if err != nil {
+		return false
+	}
+	defer k.Close()
+
+	s, _, err := k.GetStringValue("SystemProductName")
+	if err != nil {
+		return false
+	}
+	s = strings.TrimSpace(s)
+	return strings.HasPrefix(s, "Google")
+}