add escaping of comment text in webhook default JSON template

A check in image proxy for validating content type of requested images added. Modified the related tests to accommodate these changes.

.editorconfig

@@ -1,3 +1,5 @@
+root = true
+
 [*]
 indent_style = tab
 insert_final_newline = true

.github/workflows/ci-backend.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: debug if needed
         run: if [[ "$DEBUG" == "true" ]]; then env; fi
@@ -30,36 +30,44 @@ jobs:
           DEBUG: ${{secrets.DEBUG}}
 
       - name: install go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: "1.21"
 
-      - name: install golangci-lint and goveralls
+      - name: test and build backend
-        run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GITHUB_WORKSPACE v1.49.0
-          go install github.com/mattn/goveralls@latest
-
-      - name: test and lint backend
         run: |
           go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
           cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/app
         env:
           TZ: "America/Chicago"
 
-      - name: test and lint examples
+      - name: test examples
         run: |
-          go version
-          $GITHUB_WORKSPACE/golangci-lint version
           go test -race ./...
-          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+          go build -race ./...
         working-directory: backend/_example/memory_store
         env:
           TZ: "America/Chicago"
 
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          working-directory: backend/app
+
+      - name: golangci-lint on example directory
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.58
+          args: --config ../../.golangci.yml
+          working-directory: backend/_example/memory_store
+
       - name: submit coverage
-        run: goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        run: |
+          go install github.com/mattn/goveralls@latest
+          goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
         working-directory: backend
         env:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-build.yml

@@ -29,14 +29,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}

.github/workflows/ci-frontend-api.yml

@@ -20,19 +20,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -64,19 +64,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -108,19 +108,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7

.github/workflows/ci-frontend.yml

@@ -20,19 +20,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -64,19 +64,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -108,19 +108,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
@@ -155,39 +155,40 @@ jobs:
       CI_JOB_NUMBER: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7
           run_install: false
 
       - name: Check bundle size
-        uses: andresz1/size-limit-action@v1
+        uses: andresz1/size-limit-action@7313b26c76b3666c1dc41e2ca05370e201a9b7de
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           directory: ./frontend/apps/remark42
+          package_manager: pnpm
 
   test:
     name: Tests & Coverage
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16.15.1]
+        node: [16.20.2]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
       - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.4.0
         id: pnpm-install
         with:
           version: 7

.github/workflows/ci-site.yml

@@ -20,14 +20,14 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}

.github/workflows/e2e-tests.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build & run containers
         id: tests

.gitignore

@@ -25,3 +25,4 @@ compose-private.yml
 /backend/_example/*/vendor
 http-client.env.json
 /playwright-report/
+/backend/app/cmd/var

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM node:16.15.1-alpine AS frontend-deps
+FROM --platform=$BUILDPLATFORM node:16.20-alpine AS frontend-deps
 
 ARG SKIP_FRONTEND_TEST
 ARG SKIP_FRONTEND_BUILD
@@ -11,7 +11,7 @@ COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/
 RUN \
   if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
     apk add --no-cache --update git && \
-    npm i -g pnpm; \
+    npm i -g pnpm@7; \
   fi
 
 RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 \
@@ -45,7 +45,7 @@ RUN \
     echo 'Skip frontend build'; \
   fi
 
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG CI
 ARG GITHUB_REF
@@ -70,7 +70,7 @@ RUN \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
         CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
-        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov && \
         golangci-lint run --config ../.golangci.yml ./... ; \
     else \
       echo "skip backend tests and linter" \
@@ -81,7 +81,7 @@ RUN \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM umputun/baseimage:app-v1.9.2
+FROM umputun/baseimage:app-v1.13.0
 
 ARG GITHUB_SHA
 
@@ -89,7 +89,7 @@ LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
       org.opencontainers.image.description="Remark42 comment engine" \
       org.opencontainers.image.documentation="https://remark42.com/docs/getting-started/" \
       org.opencontainers.image.licenses="MIT" \
-      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
       org.opencontainers.image.title="Remark42" \
       org.opencontainers.image.url="https://remark42.com/" \
       org.opencontainers.image.revision="${GITHUB_SHA}"

Dockerfile.artifacts

@@ -1,4 +1,4 @@
-FROM node:16.15.1-alpine AS frontend-deps
+FROM node:16.20.2-alpine AS frontend-deps
 
 ENV CI=true
 
@@ -7,7 +7,7 @@ WORKDIR /srv/frontend
 COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
 COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/package.json
 
-RUN apk add --no-cache --update git && npm i -g pnpm
+RUN apk add --no-cache --update git && npm i -g pnpm@7
 RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 pnpm i
 
 FROM frontend-deps AS build-frontend
@@ -19,7 +19,7 @@ WORKDIR /srv/frontend/apps/remark42/
 COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
 RUN pnpm build
 
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ARG GITHUB_TOKEN
 ARG GITHUB_REF

backend/.golangci.yml

@@ -1,15 +1,10 @@
 run:
   timeout: 5m
-  output:
-    format: tab
-  skip-dirs:
-    - vendor
 
 linters-settings:
   govet:
-    check-shadowing: true
+    enable:
-  maligned:
+      - shadow
-    suggest-new: true
   goconst:
     min-len: 2
     min-occurrences: 2
@@ -31,29 +26,31 @@ linters-settings:
 linters:
   enable:
     - bodyclose
-    - megacheck
-    - revive
-    - govet
-    - unconvert
-    - gas
-    - gocyclo
     - dupl
+    - exportloopref
+    - gochecknoinits
+    - gocritic
+    - gocyclo
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
     - misspell
+    - nakedret
+    - prealloc
+    - revive
+    - staticcheck
+    - stylecheck
+    - typecheck
+    - unconvert
     - unparam
     - unused
-    - typecheck
-    - ineffassign
-    - stylecheck
-    - gochecknoinits
-    - exportloopref
-    - gocritic
-    - nakedret
-    - gosimple
-    - prealloc
   fast: false
   disable-all: true
 
 issues:
+  exclude-dirs:
+    - vendor
   exclude-rules:
     - text: "at least one file in a package should have a package comment"
       linters:

backend/_example/memory_store/Dockerfile

@@ -1,11 +1,11 @@
-FROM umputun/baseimage:buildgo-v1.9.2 as build-backend
+FROM umputun/baseimage:buildgo-v1.13.0 as build-backend
 
 ADD backend /build/backend
 WORKDIR /build/backend/_example/memory_store
 
 RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
 
-FROM umputun/baseimage:app-v1.9.2
+FROM umputun/baseimage:app-v1.13.0
 
 ARG GITHUB_SHA
 
@@ -13,7 +13,7 @@ LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
       org.opencontainers.image.description="Remark42 comment engine example JRPC memory store" \
       org.opencontainers.image.documentation="https://github.com/umputun/remark42/tree/master/backend/_example/memory_store" \
       org.opencontainers.image.licenses="MIT" \
-      org.opencontainers.image.source="https://github.com/umputun/remark42.git" \
+      org.opencontainers.image.source="https://github.com/umputun/remark42" \
       org.opencontainers.image.title="Remark42 JRPC example memory store" \
       org.opencontainers.image.url="https://remark42.com/" \
       org.opencontainers.image.revision="${GITHUB_SHA}"

backend/_example/memory_store/accessor/data.go

@@ -293,17 +293,17 @@ func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailE
 		defer m.mu.Unlock()
 
 		if req.Update == "" { // read detail value, no update requested
-			return m.getUserDetail(req)
+			return m.getUserDetail(req), nil
 		}
 
-		return m.setUserDetail(req)
+		return m.setUserDetail(req), nil
 	case engine.AllUserDetails:
 		// list of all details returned in case request is a read request
 		// (Update is not set) and does not have UserID or Detail set
 		if req.Update == "" && req.UserID == "" { // read list of all details
 			m.mu.Lock()
 			defer m.mu.Unlock()
-			return m.listDetails(req.Locator)
+			return m.listDetails(req.Locator), nil
 		}
 		return nil, fmt.Errorf("unsupported request with userdetail all")
 	default:
@@ -319,7 +319,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 
 	switch {
 	case req.UserDetail != "": // delete user detail
-		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+		return nil
 	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
 		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
 
@@ -332,7 +333,8 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
 				return e
 			}
 		}
-		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+		return nil
 
 	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
 		if _, ok := m.posts[req.Locator.SiteID]; !ok {
@@ -437,29 +439,29 @@ func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
 
 // getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
 // as an only element of the slice.
-func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		switch req.Detail {
 		case engine.UserEmail:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}
 		case engine.UserTelegram:
-			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}, nil
+			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}
 		}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
 // element of the slice in case of success
-func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
 	var entry metaUser
 	if meta, ok := m.metaUsers[req.UserID]; ok {
 		if meta.SiteID != req.Locator.SiteID {
-			return []engine.UserDetailEntry{}, nil
+			return []engine.UserDetailEntry{}
 		}
 		entry = meta
 	}
@@ -476,42 +478,42 @@ func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDeta
 	case engine.UserEmail:
 		entry.Details.Email = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}
 	case engine.UserTelegram:
 		entry.Details.Telegram = req.Update
 		m.metaUsers[req.UserID] = entry
-		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}, nil
+		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}
 	}
 
-	return []engine.UserDetailEntry{}, nil
+	return []engine.UserDetailEntry{}
 }
 
 // listDetails lists all available users details for given siteID
-func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+func (m *MemData) listDetails(loc store.Locator) []engine.UserDetailEntry {
 	var res []engine.UserDetailEntry
 	for _, u := range m.metaUsers {
 		if u.SiteID == loc.SiteID {
 			res = append(res, u.Details)
 		}
 	}
-	return res, nil
+	return res
 }
 
 // deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
 // deletion of the absent entry doesn't produce error.
 // Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
-func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) {
 	var entry metaUser
 	if meta, ok := m.metaUsers[userID]; ok {
 		if meta.SiteID != locator.SiteID {
-			return nil
+			return
 		}
 		entry = meta
 	}
 
 	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
 		// absent entry means that we should not do anything
-		return nil
+		return
 	}
 
 	switch userDetail {
@@ -529,7 +531,6 @@ func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDet
 	}
 
 	m.metaUsers[userID] = entry
-	return nil
 }
 
 func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {

backend/_example/memory_store/accessor/data_test.go

@@ -547,12 +547,12 @@ func TestMemData_FlagListBlocked(t *testing.T) {
 	assert.NoError(t, err)
 
 	blockedList := toBlocked(vv)
-	var blockedIds = make([]string, len(blockedList))
+	var blockedIDs = make([]string, len(blockedList))
 	for i, x := range blockedList {
-		blockedIds[i] = x.ID
+		blockedIDs[i] = x.ID
 	}
 	require.Equal(t, 2, len(blockedList), b.metaUsers)
-	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIDs)
 	t.Logf("%+v", blockedList)
 
 	// check block expiration

backend/_example/memory_store/accessor/image.go

@@ -70,6 +70,17 @@ func (m *MemImage) Load(id string) ([]byte, error) {
 	return img, nil
 }
 
+// Delete image by ID
+func (m *MemImage) Delete(id string) error {
+	m.mu.Lock()
+	// delete key from permanent and staging storage
+	delete(m.images, id)
+	delete(m.insertTime, id)
+	delete(m.imagesStaging, id)
+	m.mu.Unlock()
+	return nil
+}
+
 // Commit moves image from staging to permanent
 func (m *MemImage) Commit(id string) error {
 	m.mu.RLock()

backend/_example/memory_store/accessor/image_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 // gopher png for test, from https://golang.org/src/image/png/example_test.go
-const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+const rawGopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
 	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
 	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
 	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
@@ -38,7 +38,9 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
 	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
 	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
 
-func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNG() io.Reader {
+	return base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawGopher))
+}
 
 func TestMemImage_LoadAfterSave(t *testing.T) {
 	svc := NewMemImageStore()
@@ -57,7 +59,8 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, gopher, img)
 
-	svc.ResetCleanupTimer(id)
+	err = svc.ResetCleanupTimer(id)
+	assert.NoError(t, err)
 
 	err = svc.Commit(id)
 	assert.NoError(t, err)
@@ -70,6 +73,26 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
 	assert.Equal(t, gopher, img)
 }
 
+func TestMemImage_LoadAfterDelete(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	img, err := svc.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+	assert.Empty(t, img)
+
+	err = svc.ResetCleanupTimer(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
 func TestMemImage_CommitFail(t *testing.T) {
 	svc := NewMemImageStore()
 	err := svc.Commit("test_id")

backend/_example/memory_store/go.mod

@@ -1,42 +1,43 @@
 module github.com/umputun/remark42/memory_store
 
-go 1.17
+go 1.21
 
 require (
 	github.com/go-pkgz/jrpc v0.3.0
-	github.com/go-pkgz/lgr v0.10.4
+	github.com/go-pkgz/lgr v0.11.1
 	github.com/jessevdk/go-flags v1.5.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.9.0
-	github.com/umputun/remark42/backend v1.10.1
+	github.com/umputun/remark42/backend v1.1000.0
 )
 
 require (
 	github.com/Depado/bfchroma/v2 v2.0.0 // indirect
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/PuerkitoBio/goquery v1.9.1 // indirect
 	github.com/ajg/form v1.5.1 // indirect
-	github.com/alecthomas/chroma/v2 v2.3.0 // indirect
+	github.com/alecthomas/chroma/v2 v2.13.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/didip/tollbooth/v7 v7.0.1 // indirect
 	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.7 // indirect
+	github.com/go-chi/chi/v5 v5.0.12 // indirect
-	github.com/go-chi/render v1.0.2 // indirect
+	github.com/go-chi/render v1.0.3 // indirect
-	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
+	github.com/go-pkgz/expirable-cache v1.0.0 // indirect
-	github.com/go-pkgz/rest v1.16.0 // indirect
+	github.com/go-pkgz/rest v1.19.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.20 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rs/xid v1.4.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
-	golang.org/x/image v0.0.0-20220617043117-41969df76e82 // indirect
+	golang.org/x/image v0.15.0 // indirect
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
+	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/umputun/remark42/backend => ../../
+replace github.com/umputun/remark42/backend v1.1000.0 => ../../

backend/_example/memory_store/go.sum

@@ -1,14 +1,17 @@
 github.com/Depado/bfchroma/v2 v2.0.0 h1:IRpN9BPkNwEpR6w1ectIcNWOuhDSLx+8f1pn83fzxx8=
 github.com/Depado/bfchroma/v2 v2.0.0/go.mod h1:wFwW/Pw8Tnd0irzgO9Zxtxgzp3aPS8qBWlyadxujxmw=
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
-github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
 github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
-github.com/alecthomas/chroma/v2 v2.3.0 h1:83xfxrnjv8eK+Cf8qZDzNo3PPF9IbTWHs7z28GY6D0U=
+github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
-github.com/alecthomas/chroma/v2 v2.3.0/go.mod h1:mZxeWZlxP2Dy+/8cBob2PYd8O2DwNAzave5AY7A2eQw=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
-github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -19,29 +22,33 @@ github.com/didip/tollbooth/v7 v7.0.1 h1:TkT4sBKoQoHQFPf7blQ54iHrZiTDnr8TceU+MulV
 github.com/didip/tollbooth/v7 v7.0.1/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4=
 github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f h1:jtKwihcLmUC9BAhoJ9adCUqdSSZcOdH2KL7mPTUm2aw=
 github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f/go.mod h1:q9C80dnsuVRP2dAskjnXRNWdUJqtGgwG9wNrzt0019s=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
 github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
+github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg=
+github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
-github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
+github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
-github.com/go-pkgz/expirable-cache v0.1.0 h1:3bw0m8vlTK8qlwz5KXuygNBTkiKRTPrAGXU0Ej2AC1g=
 github.com/go-pkgz/expirable-cache v0.1.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
+github.com/go-pkgz/expirable-cache v1.0.0 h1:ns5+1hjY8hntGv8bPaQd9Gr7Jyo+Uw5SLyII40aQdtA=
+github.com/go-pkgz/expirable-cache v1.0.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs=
 github.com/go-pkgz/jrpc v0.3.0 h1:Fls38KqPsHzvp0FWfivr6cGnncC+iFBodHBqvUPY+0U=
 github.com/go-pkgz/jrpc v0.3.0/go.mod h1:MFtKs75JESiSqVicsQkgN2iDFFuCd3gVT1/vKiwRi00=
-github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
+github.com/go-pkgz/lgr v0.11.1 h1:hXFhZcznehI6imLhEa379oMOKFz7TQUmisAqb3oLOSM=
-github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
+github.com/go-pkgz/lgr v0.11.1/go.mod h1:tgDF4RXQnBfIgJqjgkv0yOeTQ3F1yewWIZkpUhHnAkU=
 github.com/go-pkgz/rest v1.15.6/go.mod h1:KUWAqbDteYGS/CiXftomQsKjtEOifXsJ36Ka0skYbmk=
-github.com/go-pkgz/rest v1.16.0 h1:3An51XJXUq1XtICHvfAEp048ZCbUHBEga3g0WEu3zK0=
+github.com/go-pkgz/rest v1.19.0 h1:FNMi5QX5dDIkuC+/e0r+CWsTuOTwUiWMRSA16Ou+9+A=
-github.com/go-pkgz/rest v1.16.0/go.mod h1:HHlLOt02NJc2sgffXBF6hYVMcRo4Gz3vjg43zTzN7VM=
+github.com/go-pkgz/rest v1.19.0/go.mod h1:Po+W6zQzpMPP6XDGLdAN2aW7UKk1IyrLSb48Lp1N3oQ=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -49,44 +56,71 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
+github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
-github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
+github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
-golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
+go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=
-golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
+golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/server/image.go

@@ -35,7 +35,6 @@ func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.R
 	}
 	err := s.img.ResetCleanupTimer(fileID)
 	return jrpc.EncodeResponse(id, nil, err)
-
 }
 
 func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
@@ -47,6 +46,16 @@ func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response)
 	return jrpc.EncodeResponse(id, value, err)
 }
 
+func (s *RPC) imgDeleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Delete(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+
+}
+
 func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
 	var fileID string
 	if err := json.Unmarshal(params, &fileID); err != nil {

backend/_example/memory_store/server/image_test.go

@@ -158,4 +158,9 @@ func TestRPC_imgInfoHndl(t *testing.T) {
 	info, err = ri.Info()
 	assert.NoError(t, err)
 	assert.False(t, info.FirstStagingImageTS.IsZero())
+
+	err = ri.Delete("test_img")
+	assert.NoError(t, err)
+	_, err = ri.Load("test_img")
+	assert.EqualError(t, err, "image test_img not found")
 }

backend/_example/memory_store/server/rpc.go

@@ -60,6 +60,7 @@ func (s *RPC) addHandlers() {
 		"save_with_id":        s.imgSaveWithIDHndl,
 		"reset_cleanup_timer": s.imgResetClnTimerHndl,
 		"load":                s.imgLoadHndl,
+		"delete":              s.imgDeleteHndl,
 		"commit":              s.imgCommitHndl,
 		"cleanup":             s.imgCleanupHndl,
 		"info":                s.imgInfoHndl,

backend/app/cmd/backup_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +18,10 @@ func TestBackup_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
 		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		fmt.Fprint(w, "blah\nblah2\n12345678\n")
 	}))
 	defer ts.Close()
@@ -34,6 +40,28 @@ func TestBackup_Execute(t *testing.T) {
 	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
 }
 
+func TestBackup_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		t.Logf("Authorization: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		require.Equal(t, "admin:", string(auth))
+		w.WriteHeader(http.StatusUnauthorized)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestBackup_ExecuteFailedStatus(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")

backend/app/cmd/cleanup_test.go

@@ -195,7 +195,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
 	})
 
-	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "DELETE", r.Method)
 		t.Log("delete ", r.URL.Path)
 		c.lock.Lock()
@@ -203,7 +203,7 @@ func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
 		c.lock.Unlock()
 	})
 
-	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/v1/admin/title/{id}", func(_ http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "PUT", r.Method)
 		t.Log("title for ", r.URL.Path)
 		c.lock.Lock()

backend/app/cmd/cmd.go

@@ -115,6 +115,9 @@ func responseError(resp *http.Response) error {
 	if e != nil {
 		body = []byte("")
 	}
+	if resp.StatusCode == http.StatusUnauthorized {
+		return fmt.Errorf("error response %q, ensure you have set ADMIN_PASSWD and provided it to the command you're running: %s", resp.Status, body)
+	}
 	return fmt.Errorf("error response %q, %s", resp.Status, body)
 }
 

backend/app/cmd/import_test.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -18,6 +20,10 @@ func TestImport_Execute(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
 		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
@@ -46,6 +52,42 @@ func TestImport_Execute(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestImport_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}
+
 func TestImport_ExecuteFailed(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")

backend/app/cmd/remap_test.go

@@ -1,9 +1,12 @@
 package cmd
 
 import (
+	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/jessevdk/go-flags"
@@ -16,6 +19,10 @@ func TestRemap_Execute(t *testing.T) {
 		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
 		assert.Equal(t, "POST", r.Method)
 		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:secret", string(auth))
 		body, err := io.ReadAll(r.Body)
 		assert.NoError(t, err)
 		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
@@ -33,3 +40,31 @@ func TestRemap_Execute(t *testing.T) {
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 }
+
+func TestRemap_ExecuteNoPassword(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "remark", r.URL.Query().Get("site"))
+		t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
+		auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
+		require.NoError(t, err)
+		assert.Equal(t, "admin:", string(auth))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
+
+		w.WriteHeader(401)
+		fmt.Fprint(w, "Unauthorized")
+	}))
+	defer ts.Close()
+
+	cmd := RemapCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
+}

backend/app/cmd/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"embed"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,7 +16,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/jrpc"
-	"github.com/go-pkgz/lcw/eventbus"
+	"github.com/go-pkgz/lcw/v2/eventbus"
 	log "github.com/go-pkgz/lgr"
 	ntf "github.com/go-pkgz/notify"
 	"github.com/golang-jwt/jwt"
@@ -27,7 +28,7 @@ import (
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 
 	"github.com/umputun/remark42/backend/app/migrator"
 	"github.com/umputun/remark42/backend/app/notify"
@@ -58,34 +59,36 @@ type ServerCommand struct {
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	Sites                      []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AnonymousVote              bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
-	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	AdminPasswd                string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	BackupLocation             string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	MaxBackupFiles             int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	LegacyImageProxy           bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
-	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MinCommentSize             int           `long:"min-comment" env:"MIN_COMMENT_SIZE" default:"0" description:"min comment size"`
-	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	MaxCommentSize             int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	MaxVotes                   int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	RestrictVoteIP             bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
-	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	DurationVoteIP             time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
-	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	LowScore                   int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	CriticalScore              int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	PositiveScore              bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	ReadOnlyAge                int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	AdminEdit        bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
+	EditDuration               time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	AdminEdit                  bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
-	Address          string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
+	Port                       int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	Address                    string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
-	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	WebRoot                    string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	UpdateLimit                float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedNames  []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
+	RestrictedWords            []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
-	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	RestrictedNames            []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
-	SimpleView       bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
+	EnableEmoji                bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
-	ProxyCORS        bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
+	SimpleView                 bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
-	AllowedHosts     []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
+	ProxyCORS                  bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
-	SubscribersOnly  bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
+	AllowedHosts               []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
-	DisableSignature bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	SubscribersOnly            bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
+	DisableSignature           bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	DisableFancyTextFormatting bool          `long:"disable-fancy-text-formatting" env:"DISABLE_FANCY_TEXT_FORMATTING" description:"disable fancy comments text formatting (replacement of quotes, dashes, fractions, etc)"`
 
 	Auth struct {
 		TTL struct {
@@ -217,14 +220,15 @@ type TelegramGroup struct {
 
 // SMTPGroup defines options for SMTP server connection, used in auth and notify modules
 type SMTPGroup struct {
-	Host      string        `long:"host" env:"HOST" description:"SMTP host"`
+	Host               string        `long:"host" env:"HOST" description:"SMTP host"`
-	Port      int           `long:"port" env:"PORT" description:"SMTP port"`
+	Port               int           `long:"port" env:"PORT" description:"SMTP port"`
-	Username  string        `long:"username" env:"USERNAME" description:"SMTP user name"`
+	Username           string        `long:"username" env:"USERNAME" description:"SMTP user name"`
-	Password  string        `long:"password" env:"PASSWORD" description:"SMTP password"`
+	Password           string        `long:"password" env:"PASSWORD" description:"SMTP password"`
-	TLS       bool          `long:"tls" env:"TLS" description:"enable TLS"`
+	TLS                bool          `long:"tls" env:"TLS" description:"enable TLS"`
-	LoginAuth bool          `long:"login_auth" env:"LOGIN_AUTH" description:"enable LOGIN auth instead of PLAIN"`
+	InsecureSkipVerify bool          `long:"insecure_skip_verify" env:"INSECURE_SKIP_VERIFY" description:"skip certificate verification"`
-	StartTLS  bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
+	LoginAuth          bool          `long:"login_auth" env:"LOGIN_AUTH" description:"enable LOGIN auth instead of PLAIN"`
-	TimeOut   time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
+	StartTLS           bool          `long:"starttls" env:"STARTTLS" description:"enable StartTLS"`
+	TimeOut            time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"SMTP TCP connection timeout"`
 }
 
 // NotifyGroup defines options for notification
@@ -500,11 +504,12 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		EditDuration:           s.EditDuration,
 		AdminEdits:             s.AdminEdit,
 		AdminStore:             adminStore,
+		MinCommentSize:         s.MinCommentSize,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
 		PositiveScore:          s.PositiveScore,
 		ImageService:           imageService,
-		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}, s.getAllowedDomains()),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
 	dataService.RestrictSameIPVotes.Enabled = s.RestrictVoteIP
@@ -525,7 +530,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	authenticator := s.getAuthenticator(dataService, avatarStore, adminStore, authRefreshCache)
 
 	telegramAuth := s.makeTelegramAuth(authenticator) // telegram auth requires TelegramAPI listener which is constructed below
-	telegramService, telegramBotUsername := s.startTelegramAuthAndNotify(ctx, telegramAuth)
+	telegramService := s.startTelegramAuthAndNotify(ctx, telegramAuth)
 
 	err = s.addAuthProviders(authenticator)
 	if err != nil {
@@ -540,7 +545,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		Cache:             loadingCache,
 		NativeImporter:    &migrator.Native{DataStore: dataService},
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
-		WordPressImporter: &migrator.WordPress{DataStore: dataService},
+		WordPressImporter: &migrator.WordPress{DataStore: dataService, DisableFancyTextFormatting: s.DisableFancyTextFormatting},
 		CommentoImporter:  &migrator.Commento{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
 		URLMapperMaker:    migrator.NewURLMapper,
@@ -575,33 +580,34 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	}
 
 	srv := &api.Rest{
-		Version:             s.Revision,
+		Version:                    s.Revision,
-		DataService:         dataService,
+		DataService:                dataService,
-		WebRoot:             s.WebRoot,
+		WebRoot:                    s.WebRoot,
-		WebFS:               webFS,
+		WebFS:                      webFS,
-		RemarkURL:           s.RemarkURL,
+		RemarkURL:                  s.RemarkURL,
-		ImageProxy:          imgProxy,
+		ImageProxy:                 imgProxy,
-		CommentFormatter:    commentFormatter,
+		CommentFormatter:           commentFormatter,
-		Migrator:            migr,
+		Migrator:                   migr,
-		ReadOnlyAge:         s.ReadOnlyAge,
+		ReadOnlyAge:                s.ReadOnlyAge,
-		SharedSecret:        s.SharedSecret,
+		SharedSecret:               s.SharedSecret,
-		Authenticator:       authenticator,
+		Authenticator:              authenticator,
-		Cache:               loadingCache,
+		Cache:                      loadingCache,
-		NotifyService:       notifyService,
+		NotifyService:              notifyService,
-		TelegramService:     telegramService,
+		TelegramService:            telegramService,
-		SSLConfig:           sslConfig,
+		SSLConfig:                  sslConfig,
-		UpdateLimiter:       s.UpdateLimit,
+		UpdateLimiter:              s.UpdateLimit,
-		ImageService:        imageService,
+		ImageService:               imageService,
-		EmailNotifications:  contains("email", s.Notify.Users),
+		EmailNotifications:         contains("email", s.Notify.Users),
-		TelegramBotUsername: telegramBotUsername,
+		TelegramNotifications:      contains("telegram", s.Notify.Users) && telegramService != nil,
-		EmojiEnabled:        s.EnableEmoji,
+		EmojiEnabled:               s.EnableEmoji,
-		AnonVote:            s.AnonymousVote && s.RestrictVoteIP,
+		AnonVote:                   s.AnonymousVote && s.RestrictVoteIP,
-		SimpleView:          s.SimpleView,
+		SimpleView:                 s.SimpleView,
-		ProxyCORS:           s.ProxyCORS,
+		ProxyCORS:                  s.ProxyCORS,
-		AllowedAncestors:    s.AllowedHosts,
+		AllowedAncestors:           s.AllowedHosts,
-		SendJWTHeader:       s.Auth.SendJWTHeader,
+		SendJWTHeader:              s.Auth.SendJWTHeader,
-		SubscribersOnly:     s.SubscribersOnly,
+		SubscribersOnly:            s.SubscribersOnly,
-		DisableSignature:    s.DisableSignature,
+		DisableSignature:           s.DisableSignature,
+		DisableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -633,6 +639,47 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	}, nil
 }
 
+// Extract domains from s.AllowedHosts and second level domain from s.RemarkURL.
+// It can be and IP like http://127.0.0.1 in which case we need to use whole IP as domain
+// Beware, if s.RemarkURL is in third-level domain like https://example.co.uk, co.uk will be returned.
+func (s *ServerCommand) getAllowedDomains() []string {
+	rawDomains := s.AllowedHosts
+	rawDomains = append(rawDomains, s.RemarkURL)
+	allowedDomains := []string{}
+	for _, rawURL := range rawDomains {
+		// case of 'self' AllowedHosts, which is not a valid rawURL name
+		if rawURL == "self" || rawURL == "'self'" || rawURL == "\"self\"" {
+			continue
+		}
+		// AllowedHosts usually don't have https:// prefix, so we're adding it just to make parsing below work the same way as for RemarkURL
+		if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
+			rawURL = "https://" + rawURL
+		}
+		parsedURL, err := url.Parse(rawURL)
+		if err != nil {
+			log.Printf("[WARN] failed to parse URL %s for TitleExtract whitelist: %v", rawURL, err)
+			continue
+		}
+		domain := parsedURL.Hostname()
+
+		if domain == "" || // don't add empty domain as it will allow everything to be extracted
+			(len(strings.Split(domain, ".")) < 2 && // don't allow single-word domains like "com"
+				domain != "localhost") { // localhost is an exceptional single-word domain which is allowed
+			continue
+		}
+
+		// Only for RemarkURL if domain is not IP and has more than two levels, extract second level domain.
+		// For AllowedHosts we don't do this as they are exact list of domains which can host comments, but
+		// RemarkURL might be on a subdomain and we must allow parent domain to be used for TitleExtract.
+		if rawURL == s.RemarkURL && net.ParseIP(domain) == nil && len(strings.Split(domain, ".")) > 2 {
+			domain = strings.Join(strings.Split(domain, ".")[len(strings.Split(domain, "."))-2:], ".")
+		}
+
+		allowedDomains = append(allowedDomains, domain)
+	}
+	return allowedDomains
+}
+
 // Run all application objects
 func (a *serverApp) run(ctx context.Context) error {
 	if a.AdminPasswd != "" {
@@ -822,27 +869,28 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 
 func (s *ServerCommand) makeCache() (LoadingCache, error) {
 	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
+	o := cache.NewOpts[[]byte]()
 	switch s.Cache.Type {
 	case "redis_pub_sub":
 		redisPubSub, err := eventbus.NewRedisPubSub(s.Cache.RedisAddr, "remark42-cache")
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization, redis PubSub initialisation: %w", err)
 		}
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items), cache.EventBus(redisPubSub))
+			o.MaxKeys(s.Cache.Max.Items), o.EventBus(redisPubSub))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "mem":
-		backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+		backend, err := cache.NewLruCache(o.MaxCacheSize(s.Cache.Max.Size), o.MaxValSize(s.Cache.Max.Value),
-			cache.MaxKeys(s.Cache.Max.Items))
+			o.MaxKeys(s.Cache.Max.Items))
 		if err != nil {
 			return nil, fmt.Errorf("cache backend initialization: %w", err)
 		}
-		return cache.NewScache(backend), nil
+		return cache.NewScache[[]byte](backend), nil
 	case "none":
-		return cache.NewScache(&cache.Nop{}), nil
+		return cache.NewScache[[]byte](&cache.Nop[[]byte]{}), nil
 	}
 	return nil, fmt.Errorf("unsupported cache type %s", s.Cache.Type)
 }
@@ -857,9 +905,10 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 	if s.Auth.Apple.CID != "" && s.Auth.Apple.TID != "" && s.Auth.Apple.KID != "" {
 		err := authenticator.AddAppleProvider(
 			provider.AppleConfig{
-				ClientID: s.Auth.Apple.CID,
+				ClientID:     s.Auth.Apple.CID,
-				TeamID:   s.Auth.Apple.TID,
+				TeamID:       s.Auth.Apple.TID,
-				KeyID:    s.Auth.Apple.KID,
+				KeyID:        s.Auth.Apple.KID,
+				ResponseMode: "query", // default is form_post which wouldn't work here
 			},
 			provider.LoadApplePrivateKeyFromFile(s.Auth.Apple.PrivateKeyFilePath),
 		)
@@ -909,18 +958,19 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
 
 	if s.Auth.Email.Enable {
 		params := sender.EmailParams{
-			Host:         s.SMTP.Host,
+			Host:               s.SMTP.Host,
-			Port:         s.SMTP.Port,
+			Port:               s.SMTP.Port,
-			SMTPUserName: s.SMTP.Username,
+			SMTPUserName:       s.SMTP.Username,
-			SMTPPassword: s.SMTP.Password,
+			SMTPPassword:       s.SMTP.Password,
-			TimeOut:      s.SMTP.TimeOut,
+			TimeOut:            s.SMTP.TimeOut,
-			StartTLS:     s.SMTP.StartTLS,
+			StartTLS:           s.SMTP.StartTLS,
-			LoginAuth:    s.SMTP.LoginAuth,
+			LoginAuth:          s.SMTP.LoginAuth,
-			TLS:          s.SMTP.TLS,
+			TLS:                s.SMTP.TLS,
-			Charset:      "UTF-8",
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
-			From:         s.Auth.Email.From,
+			Charset:            "UTF-8",
-			Subject:      s.Auth.Email.Subject,
+			From:               s.Auth.Email.From,
-			ContentType:  s.Auth.Email.ContentType,
+			Subject:            s.Auth.Email.Subject,
+			ContentType:        s.Auth.Email.ContentType,
 		}
 		sndr := sender.NewEmailClient(params, log.Default())
 		tmpl, err := templates.Read(s.Auth.Email.MsgTemplate)
@@ -1064,16 +1114,17 @@ func (s *ServerCommand) makeNotifyDestinations(authenticator *auth.Service) ([]n
 			emailParams.AdminEmails = s.Admin.Shared.Email
 		}
 		smtpParams := ntf.SMTPParams{
-			Host:        s.SMTP.Host,
+			Host:               s.SMTP.Host,
-			Port:        s.SMTP.Port,
+			Port:               s.SMTP.Port,
-			TLS:         s.SMTP.TLS,
+			TLS:                s.SMTP.TLS,
-			StartTLS:    s.SMTP.StartTLS,
+			StartTLS:           s.SMTP.StartTLS,
-			LoginAuth:   s.SMTP.LoginAuth,
+			InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
-			Username:    s.SMTP.Username,
+			LoginAuth:          s.SMTP.LoginAuth,
-			Password:    s.SMTP.Password,
+			Username:           s.SMTP.Username,
-			TimeOut:     s.SMTP.TimeOut,
+			Password:           s.SMTP.Password,
-			ContentType: "text/html",
+			TimeOut:            s.SMTP.TimeOut,
-			Charset:     "UTF-8",
+			ContentType:        "text/html",
+			Charset:            "UTF-8",
 		}
 		emailService, err := notify.NewEmail(emailParams, smtpParams)
 		if err != nil {
@@ -1174,7 +1225,7 @@ func (s *ServerCommand) getAuthenticator(ds *service.DataStore, avas avatar.Stor
 			return c
 		}),
 		AdminPasswd: s.AdminPasswd,
-		Validator: token.ValidatorFunc(func(token string, claims token.Claims) bool { // check on each auth call (in middleware)
+		Validator: token.ValidatorFunc(func(_ string, claims token.Claims) bool { // check on each auth call (in middleware)
 			if claims.User == nil {
 				return false
 			}
@@ -1211,20 +1262,15 @@ func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
 
 // startTelegramAuthAndNotify initializes telegram notify and auth Telegram Bot listen loop.
 // Does nothing if telegram auth and notifications are disabled.
-// Doesn't return telegram bot username if user notifications are disabled, as that is the way frontend knows they are enabled.
+func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram) {
-func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegramAuth providers.TGUpdatesReceiver) (tg *notify.Telegram, telegramBotUsername string) {
 	if !contains("telegram", s.Notify.Users) && !contains("telegram", s.Notify.Admins) && !s.Auth.Telegram {
-		return nil, ""
+		return nil
 	}
 
 	var err error
 	if tg, err = s.makeTelegramNotify(); err != nil {
 		log.Printf("[WARN] failed to make telegram notify service, %s", err)
-		return nil, ""
+		return nil
-	}
-
-	if contains("telegram", s.Notify.Users) {
-		telegramBotUsername = tg.GetBotUsername()
 	}
 
 	telegramReceivers := []providers.TGUpdatesReceiver{tg}
@@ -1234,7 +1280,7 @@ func (s *ServerCommand) startTelegramAuthAndNotify(ctx context.Context, telegram
 	// start bot messages receiver for both notify and auth services
 	go providers.DispatchTelegramUpdates(ctx, tg, telegramReceivers, time.Second*5)
 
-	return tg, telegramBotUsername
+	return tg
 }
 
 // splitAtCommas split s at commas, ignoring commas in strings.
@@ -1281,11 +1327,12 @@ func splitAtCommas(s string) []string {
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	cache.LoadingCache
+	cache.LoadingCache[token.Claims]
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	expirableCache, _ := cache.NewExpirableCache(cache.TTL(5 * time.Minute))
+	o := cache.NewOpts[token.Claims]()
+	expirableCache, _ := cache.NewExpirableCache(o.TTL(5 * time.Minute))
 	return &authRefreshCache{LoadingCache: expirableCache}
 }
 
@@ -1296,5 +1343,5 @@ func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 
 // Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	_, _ = c.LoadingCache.Get(key.(string), func() (interface{}, error) { return value, nil })
+	_, _ = c.LoadingCache.Get(key.(string), func() (token.Claims, error) { return value.(token.Claims), nil })
 }

backend/app/cmd/server_test.go

@@ -250,7 +250,7 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -612,7 +612,7 @@ func TestServerAuthHooks(t *testing.T) {
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID:   "dev",
+			ID:   "github_dev",
 			Name: "developer one",
 		},
 	}
@@ -651,14 +651,14 @@ func TestServerAuthHooks(t *testing.T) {
 	require.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
 
-	// block user dev as admin
+	// block user github_dev as admin
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), http.NoBody)
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/github_dev?site=remark&block=1&ttl=10d", port), http.NoBody)
 	assert.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "user github_dev blocked")
 	b, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
@@ -726,6 +726,28 @@ func Test_splitAtCommas(t *testing.T) {
 	}
 }
 
+func Test_getAllowedDomains(t *testing.T) {
+	tbl := []struct {
+		s              ServerCommand
+		allowedDomains []string
+	}{
+		// correct example, parsed and returned as allowed domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "https://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://remark42.example.org"}}, []string{"example.org"}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "http://localhost"}}, []string{"localhost"}},
+		// incorrect URLs, so Hostname is empty but returned list doesn't include empty string as it would allow any domain
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "bad hostname"}}, []string{}},
+		{ServerCommand{AllowedHosts: []string{}, CommonOpts: CommonOpts{RemarkURL: "not_a_hostname"}}, []string{}},
+		// test removal of 'self', multiple AllowedHosts. No deduplication is expected
+		{ServerCommand{AllowedHosts: []string{"'self'", "example.org", "test.example.org", "remark42.com"}, CommonOpts: CommonOpts{RemarkURL: "https://example.org"}}, []string{"example.org", "test.example.org", "remark42.com", "example.org"}},
+	}
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			assert.Equal(t, tt.allowedDomains, tt.s.getAllowedDomains())
+		})
+	}
+}
+
 func chooseRandomUnusedPort() (port int) {
 	for i := 0; i < 10; i++ {
 		port = 40000 + int(rand.Int31n(10000))
@@ -821,12 +843,15 @@ func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Cont
 	ctx, cancel := context.WithCancel(context.Background())
 	app, err := cmd.newServerApp(ctx)
 	require.NoError(t, err)
-
-	rand.Seed(time.Now().UnixNano())
 	return app, ctx, cancel
 }
 
 func TestMain(m *testing.M) {
 	// ignore is added only for GitHub Actions, can't reproduce locally
-	goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
+	goleak.VerifyTestMain(
+		m,
+		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -23,7 +24,8 @@ type Opts struct {
 	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
 	RemapCmd   cmd.RemapCommand   `command:"remap"`
 
-	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	// SharedSecret is only used in server command, but defined for all commands for historical reasons
 	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"the shared secret key used to sign JWT, should be a random, long, hard-to-guess string"`
 
 	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
@@ -54,11 +56,11 @@ func main() {
 	}
 
 	if _, err := p.Parse(); err != nil {
-		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
+		var flagsErr *flags.Error
+		if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp {
 			os.Exit(0)
-		} else {
-			os.Exit(1)
 		}
+		os.Exit(1)
 	}
 }
 
@@ -97,7 +99,7 @@ func getDump() string {
 	return string(stacktrace[:length])
 }
 
-// nolint:gochecknoinits // can't avoid it in this place
+//nolint:gochecknoinits // can't avoid it in this place
 func init() {
 	// catch SIGQUIT and print stack traces
 	sigChan := make(chan os.Signal, 1)

backend/app/main_test.go

@@ -64,7 +64,7 @@ func TestMain_WithWebhook(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	var webhookSent int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
 		atomic.StoreInt32(&webhookSent, 1)
 		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
 
@@ -157,5 +157,7 @@ func TestMain(m *testing.M) {
 		m,
 		goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
 		goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
 	)
 }

backend/app/migrator/commento.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"time"
 
 	"github.com/umputun/remark42/backend/app/store"
@@ -110,16 +111,28 @@ func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				continue
 			}
 
+			parentID := comment.ParentHex
+			// comments with ParentHex == "root" are top-level comments
+			if parentID == "root" {
+				parentID = ""
+			}
+
+			commentURL, e := url.JoinPath("https://", comment.Domain, comment.Path)
+			if e != nil {
+				log.Printf("[WARN] can't construct comment URL in commento import, %s", err.Error())
+			}
+			log.Printf("[ERROR] commentoURL: %s", commentURL)
+
 			c := store.Comment{
 				ID: comment.CommentHex,
 				Locator: store.Locator{
-					URL:    comment.Path,
+					URL:    commentURL,
 					SiteID: siteID,
 				},
 				User:      u,
 				Text:      comment.Markdown,
 				Timestamp: comment.CreationDate,
-				ParentID:  comment.ParentHex,
+				ParentID:  parentID,
 				Imported:  true,
 			}
 

backend/app/migrator/native_test.go

@@ -163,7 +163,7 @@ func TestNative_ImportManyWithError(t *testing.T) {
 	buf := &bytes.Buffer{}
 	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
 	for i := 0; i < 100; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
+		fmt.Fprintf(buf, goodRec, i)
 	}
 	buf.WriteString("{}\n")
 	buf.WriteString("{}\n")

backend/app/migrator/testdata/commento.json

@@ -4,7 +4,7 @@
 	{
 	  "commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "anonymous",
 	  "markdown": "Example comment created by user.",
 	  "html": "",
@@ -18,7 +18,7 @@
 	{
 	  "commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 	  "markdown": "Example 2 comment created by user.",
 	  "html": "",
@@ -32,7 +32,7 @@
 	{
 	  "commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
 	  "domain": "example.com",
-	  "url": "https://example.com/blog/post/1",
+	  "url": "/blog/post/1",
 	  "commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
 	  "markdown": "Great reply!",
 	  "html": "",

backend/app/migrator/wordpress.go

@@ -16,7 +16,8 @@ const wpTimeLayout = "2006-01-02 15:04:05"
 
 // WordPress implements Importer from WP xml
 type WordPress struct {
-	DataStore Store
+	DataStore                  Store
+	DisableFancyTextFormatting bool
 }
 
 type wpItem struct {
@@ -138,7 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								ParentID:  comment.PID,
 								Imported:  true,
 							}
-							commentsCh <- commentFormatter.Format(c)
+							commentsCh <- commentFormatter.Format(c, w.DisableFancyTextFormatting)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
 								log.Printf("[DEBUG] processed %d comments", stats.inpComments)

backend/app/migrator/wordpress_test.go

@@ -24,7 +24,7 @@ func TestWordPress_Import(t *testing.T) {
 
 	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 	defer dataStore.Close()
-	wp := WordPress{DataStore: &dataStore}
+	wp := WordPress{DataStore: &dataStore, DisableFancyTextFormatting: false}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
@@ -41,7 +41,7 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
-	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.Equal(t, "<p>«Mekkatorque» was over in that tent up to the right</p>\n", c.Text)
 	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
@@ -54,6 +54,18 @@ func TestWordPress_Import(t *testing.T) {
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
+
+	// test with DisableFancyTextFormatting
+	wp = WordPress{DataStore: &dataStore, DisableFancyTextFormatting: true}
+	size, err = wp.Import(strings.NewReader(xmlTestWP), siteID)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, size)
+
+	last, err = dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
+
+	assert.Equal(t, "<p>&#34;Mekkatorque&#34; was over in that tent up to the right</p>\n", last[0].Text)
 }
 
 func TestWordPress_Convert(t *testing.T) {
@@ -247,7 +259,7 @@ var xmlTestWP = `
 			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
 			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
 			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
-			<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
+			<wp:comment_content><![CDATA["Mekkatorque" was over in that tent up to the right]]></wp:comment_content>
 			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
 			<wp:comment_type><![CDATA[]]></wp:comment_type>
 			<wp:comment_parent>13</wp:comment_parent>

backend/app/notify/webhook.go

@@ -3,6 +3,7 @@ package notify
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"text/template"
 	"time"
@@ -12,7 +13,7 @@ import (
 )
 
 const (
-	webhookDefaultTemplate = `{"text": "{{.Text}}"}`
+	webhookDefaultTemplate = `{"text": {{.Text | escapeJSONString}}}`
 )
 
 // WebhookParams contain settings for webhook notifications
@@ -49,7 +50,7 @@ func NewWebhook(params WebhookParams) (*Webhook, error) {
 		params.Template = webhookDefaultTemplate
 	}
 
-	payloadTmpl, err := template.New("webhook").Parse(params.Template)
+	payloadTmpl, err := template.New("webhook").Funcs(template.FuncMap{"escapeJSONString": escapeJSONString}).Parse(params.Template)
 	if err != nil {
 		return nil, fmt.Errorf("unable to parse webhook template: %w", err)
 	}
@@ -82,3 +83,12 @@ func (w *Webhook) SendVerification(_ context.Context, _ VerificationRequest) err
 func (w *Webhook) String() string {
 	return fmt.Sprintf("%s to %s", w.Webhook.String(), w.url)
 }
+
+// escapeJSONString escapes string for JSON
+func escapeJSONString(s string) (string, error) {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

backend/app/notify/webhook_test.go

@@ -2,6 +2,9 @@ package notify
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -34,6 +37,32 @@ func TestWebhook_NewWebhook(t *testing.T) {
 	assert.Contains(t, err.Error(), "unable to parse webhook template")
 }
 
+// https://github.com/umputun/remark42/issues/1791
+func TestWebhook_ReceiveValidJSON(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/webhook-notify")
+		assert.Equal(t, "POST", r.Method)
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		t.Log("received body", string(body))
+		assert.JSONEq(t, `{"text": "<p>testme</p>\n"}`, string(body))
+	}))
+	defer ts.Close()
+
+	wh, err := NewWebhook(WebhookParams{
+		URL:     ts.URL + "/webhook-notify",
+		Headers: []string{"Content-Type:application/json,text/plain"},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, wh)
+
+	f := store.NewCommentFormatter()
+	c := store.Comment{Text: f.FormatText("testme", false), ParentID: "1", ID: "999"}
+
+	err = wh.Send(context.Background(), Request{Comment: c})
+	assert.NoError(t, err)
+}
+
 func TestWebhook_Send(t *testing.T) {
 	wh, err := NewWebhook(WebhookParams{
 		URL:     "bad-url",

backend/app/rest/api/admin.go

@@ -9,7 +9,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 

backend/app/rest/api/admin_test.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
 	"github.com/stretchr/testify/assert"
@@ -111,7 +111,7 @@ func TestAdmin_Title(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"127.0.0.1"})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
 			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
@@ -348,7 +348,7 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
 	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
-	srv.pubRest.cache = cache.NewScache(cache.NewNopCache()) // TODO: with lru cache it won't be refreshed and invalidated for long
+	srv.pubRest.cache = cache.NewScache[[]byte](cache.NewNopCache[[]byte]()) // TODO: with lru cache it won't be refreshed and invalidated for long
 	// time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
@@ -513,6 +513,7 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
 	assert.Error(t, err)
 
+	// test format "tree"
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	comments := commentsWithInfo{}
@@ -521,6 +522,16 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 	assert.True(t, comments.Info.ReadOnly)
 	t.Logf("%+v", comments)
+
+	// test format "plain"
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah")
+	assert.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
 }
 
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
@@ -743,9 +754,9 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	defer teardown()
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "provider1_user1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "provider1_user2"}}
 
 	_, err := srv.DataService.Create(c1)
 	assert.NoError(t, err)
@@ -768,13 +779,13 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
-			Id:        "1234567",
+			Id:        "provider1_1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
 		},
 		User: &token.User{
-			ID: "user1",
+			ID: "provider1_user1",
 			Attributes: map[string]interface{}{
 				"delete_me": true,
 			},

backend/app/rest/api/migrator.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 
@@ -136,7 +136,6 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
 		w.Header().Set("Content-Type", "application/gzip")
 		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
 		gzWriter := gzip.NewWriter(w)
 		defer func() {
 			if e := gzWriter.Close(); e != nil {
@@ -163,7 +162,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "remap failed, bad given rules", rest.ErrDecode)
 		return
 	}
-	defer r.Body.Close()
+	defer r.Body.Close() //nolint gosec // we don't care about response body
 
 	// start remap procedure with mapper
 	go func() {

backend/app/rest/api/migrator_test.go

@@ -9,6 +9,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -49,6 +50,22 @@ func TestMigrator_Import(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
@@ -84,6 +101,22 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
@@ -108,6 +141,22 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 3, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://realmenweardress.es/2010/07/do-you-rp/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 3, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments), "2 comments with 1 reply")
 }
 
 func TestMigrator_ImportFromCommento(t *testing.T) {
@@ -115,7 +164,7 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1,"comments":[{"commentHex":"7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
-"domain":"example.com","url":"https://example.com/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
+"domain":"example.com","url":"/blog/post/1","commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "markdown":"Example content","html":"","parentHex":"root","score":0,"state":"approved","creationDate":"2021-03-17T12:09:47.722181Z",
 "direction":0,"deleted":false}],"commenters":[{"commenterHex":"a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
 "email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
@@ -137,6 +186,63 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/blog/post/1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+}
+
+func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	r, err := os.Open("testdata/commento.json")
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
+	assert.NoError(t, err)
+	req.Header.Add("Content-Type", "application/json; charset=utf-8")
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+	assert.NoError(t, resp.Body.Close())
+
+	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 7, len(comments.Comments))
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 7, comments.Info.Count)
+	require.Equal(t, 5, len(comments.Comments), "five comments with two replies")
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
@@ -197,6 +303,20 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 50, comments.Info.Count)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
@@ -236,6 +356,14 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
 
 	waitForMigrationCompletion(t, ts)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
+	require.Equal(t, http.StatusOK, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_Export(t *testing.T) {
@@ -263,6 +391,16 @@ func TestMigrator_Export(t *testing.T) {
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
 	waitForMigrationCompletion(t, ts)
 
+	// export wrong site, should result in error
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=test", http.NoBody)
+	require.NoError(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+
 	// check file mode
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", http.NoBody)
 	require.NoError(t, err)
@@ -339,6 +477,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
@@ -347,6 +486,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// we want remap urls to another domain - www.remark42.com
@@ -364,6 +504,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
+	require.False(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 2, comments.Info.Count)
+	require.Equal(t, 2, len(comments.Comments))
 	require.False(t, comments.Info.ReadOnly)
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
@@ -372,6 +522,16 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
+	require.True(t, comments.Info.ReadOnly)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo-another/")
+	require.Equal(t, http.StatusOK, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	require.NoError(t, err)
+	require.Equal(t, 1, comments.Info.Count)
+	require.Equal(t, 1, len(comments.Comments))
 	require.True(t, comments.Info.ReadOnly)
 
 	// should find nothing from previous url
@@ -381,6 +541,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
 	require.Equal(t, http.StatusOK, code)
@@ -388,6 +549,7 @@ func TestMigrator_Remap(t *testing.T) {
 	err = json.Unmarshal([]byte(res), &comments)
 	require.NoError(t, err)
 	require.Equal(t, 0, comments.Info.Count)
+	require.Equal(t, 0, len(comments.Comments))
 }
 
 func TestMigrator_RemapReject(t *testing.T) {

backend/app/rest/api/rest.go

@@ -22,7 +22,7 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/logger"
@@ -59,26 +59,28 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter       float64
+	UpdateLimiter              float64
-	EmailNotifications  bool
+	EmailNotifications         bool
-	TelegramBotUsername string
+	TelegramNotifications      bool
-	EmojiEnabled        bool
+	EmojiEnabled               bool
-	SimpleView          bool
+	SimpleView                 bool
-	ProxyCORS           bool
+	ProxyCORS                  bool
-	SendJWTHeader       bool
+	SendJWTHeader              bool
-	AllowedAncestors    []string // sets Content-Security-Policy "frame-ancestors ..."
+	AllowedAncestors           []string // sets Content-Security-Policy "frame-ancestors ..."
-	SubscribersOnly     bool
+	SubscribersOnly            bool
-	DisableSignature    bool // prevent signature from being added to headers
+	DisableSignature           bool // prevent signature from being added to headers
+	DisableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	pubRest   public
+	pubRest          public
-	privRest  private
+	privRest         private
-	adminRest admin
+	adminRest        admin
-	rssRest   rss
+	rssRest          rss
+	openRouteLimiter float64
 }
 
 // LoadingCache defines interface for caching
@@ -89,7 +91,7 @@ type LoadingCache interface {
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
-
+const openRouteLimiter = 10     // limit for open routes
 const lastCommentsScope = "last"
 
 type commentsWithInfo struct {
@@ -97,6 +99,11 @@ type commentsWithInfo struct {
 	Info     store.PostInfo  `json:"info,omitempty"`
 }
 
+type treeWithInfo struct {
+	*service.Tree
+	Info store.PostInfo `json:"info,omitempty"`
+}
+
 // Run the lister and request's router, activate rest server
 func (s *Rest) Run(address string, port int) {
 	if address == "*" {
@@ -192,6 +199,10 @@ func (s *Rest) makeHTTPServer(address string, port int, router http.Handler) *ht
 }
 
 func (s *Rest) routes() chi.Router {
+	if s.openRouteLimiter == 0 {
+		// set the default open route limiter. Just a safety measure as it should be set by Run method anyway
+		s.openRouteLimiter = openRouteLimiter
+	}
 	router := chi.NewRouter()
 	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
 	if !s.DisableSignature {
@@ -251,7 +262,7 @@ func (s *Rest) routes() chi.Router {
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(middleware.Timeout(30 * time.Second))
-			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.openRouteLimiter, nil)))
 			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/config", s.configCtrl)
 			ropen.Get("/find", s.pubRest.findCommentsCtrl)
@@ -369,16 +380,17 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 	}
 
 	privGrp := private{
-		dataService:      s.DataService,
+		dataService:                s.DataService,
-		cache:            s.Cache,
+		cache:                      s.Cache,
-		imageService:     s.ImageService,
+		imageService:               s.ImageService,
-		commentFormatter: s.CommentFormatter,
+		commentFormatter:           s.CommentFormatter,
-		readOnlyAge:      s.ReadOnlyAge,
+		readOnlyAge:                s.ReadOnlyAge,
-		authenticator:    s.Authenticator,
+		authenticator:              s.Authenticator,
-		notifyService:    s.NotifyService,
+		notifyService:              s.NotifyService,
-		telegramService:  s.TelegramService,
+		telegramService:            s.TelegramService,
-		remarkURL:        s.RemarkURL,
+		remarkURL:                  s.RemarkURL,
-		anonVote:         s.AnonVote,
+		anonVote:                   s.AnonVote,
+		disableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	admGrp := admin{
@@ -414,44 +426,46 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 	emails, _ := s.DataService.AdminStore.Email(siteID)
 
 	cnf := struct {
-		Version             string   `json:"version"`
+		Version               string   `json:"version"`
-		EditDuration        int      `json:"edit_duration"`
+		EditDuration          int      `json:"edit_duration"`
-		AdminEdit           bool     `json:"admin_edit"`
+		AdminEdit             bool     `json:"admin_edit"`
-		MaxCommentSize      int      `json:"max_comment_size"`
+		MinCommentSize        int      `json:"min_comment_size"`
-		Admins              []string `json:"admins"`
+		MaxCommentSize        int      `json:"max_comment_size"`
-		AdminEmail          string   `json:"admin_email"`
+		Admins                []string `json:"admins"`
-		Auth                []string `json:"auth_providers"`
+		AdminEmail            string   `json:"admin_email"`
-		AnonVote            bool     `json:"anon_vote"`
+		Auth                  []string `json:"auth_providers"`
-		LowScore            int      `json:"low_score"`
+		AnonVote              bool     `json:"anon_vote"`
-		CriticalScore       int      `json:"critical_score"`
+		LowScore              int      `json:"low_score"`
-		PositiveScore       bool     `json:"positive_score"`
+		CriticalScore         int      `json:"critical_score"`
-		ReadOnlyAge         int      `json:"readonly_age"`
+		PositiveScore         bool     `json:"positive_score"`
-		MaxImageSize        int      `json:"max_image_size"`
+		ReadOnlyAge           int      `json:"readonly_age"`
-		EmailNotifications  bool     `json:"email_notifications"`
+		MaxImageSize          int      `json:"max_image_size"`
-		TelegramBotUsername string   `json:"telegram_bot_username"`
+		EmailNotifications    bool     `json:"email_notifications"`
-		EmojiEnabled        bool     `json:"emoji_enabled"`
+		TelegramNotifications bool     `json:"telegram_notifications"`
-		SimpleView          bool     `json:"simple_view"`
+		EmojiEnabled          bool     `json:"emoji_enabled"`
-		SendJWTHeader       bool     `json:"send_jwt_header"`
+		SimpleView            bool     `json:"simple_view"`
-		SubscribersOnly     bool     `json:"subscribers_only"`
+		SendJWTHeader         bool     `json:"send_jwt_header"`
+		SubscribersOnly       bool     `json:"subscribers_only"`
 	}{
-		Version:             s.Version,
+		Version:               s.Version,
-		EditDuration:        int(s.DataService.EditDuration.Seconds()),
+		EditDuration:          int(s.DataService.EditDuration.Seconds()),
-		AdminEdit:           s.DataService.AdminEdits,
+		AdminEdit:             s.DataService.AdminEdits,
-		MaxCommentSize:      s.DataService.MaxCommentSize,
+		MinCommentSize:        s.DataService.MinCommentSize,
-		Admins:              admins,
+		MaxCommentSize:        s.DataService.MaxCommentSize,
-		AdminEmail:          emails,
+		Admins:                admins,
-		LowScore:            s.ScoreThresholds.Low,
+		AdminEmail:            emails,
-		CriticalScore:       s.ScoreThresholds.Critical,
+		LowScore:              s.ScoreThresholds.Low,
-		PositiveScore:       s.DataService.PositiveScore,
+		CriticalScore:         s.ScoreThresholds.Critical,
-		ReadOnlyAge:         s.ReadOnlyAge,
+		PositiveScore:         s.DataService.PositiveScore,
-		MaxImageSize:        s.ImageService.MaxSize,
+		ReadOnlyAge:           s.ReadOnlyAge,
-		EmailNotifications:  s.EmailNotifications,
+		MaxImageSize:          s.ImageService.MaxSize,
-		TelegramBotUsername: s.TelegramBotUsername,
+		EmailNotifications:    s.EmailNotifications,
-		EmojiEnabled:        s.EmojiEnabled,
+		TelegramNotifications: s.TelegramNotifications,
-		AnonVote:            s.AnonVote,
+		EmojiEnabled:          s.EmojiEnabled,
-		SimpleView:          s.SimpleView,
+		AnonVote:              s.AnonVote,
-		SendJWTHeader:       s.SendJWTHeader,
+		SimpleView:            s.SimpleView,
-		SubscribersOnly:     s.SubscribersOnly,
+		SendJWTHeader:         s.SendJWTHeader,
+		SubscribersOnly:       s.SubscribersOnly,
 	}
 
 	cnf.Auth = []string{}

backend/app/rest/api/rest_private.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"crypto/sha1" //nolint:gosec //not used for security
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"io"
@@ -17,7 +18,7 @@ import (
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/golang-jwt/jwt"
@@ -33,16 +34,17 @@ import (
 )
 
 type private struct {
-	dataService      privStore
+	dataService                privStore
-	cache            LoadingCache
+	cache                      LoadingCache
-	readOnlyAge      int
+	readOnlyAge                int
-	commentFormatter *store.CommentFormatter
+	commentFormatter           *store.CommentFormatter
-	imageService     *image.Service
+	imageService               *image.Service
-	notifyService    *notify.Service
+	notifyService              *notify.Service
-	authenticator    *auth.Service
+	authenticator              *auth.Service
-	telegramService  telegramService
+	telegramService            telegramService
-	remarkURL        string
+	remarkURL                  string
-	anonVote         bool
+	anonVote                   bool
+	disableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 }
 
 // telegramService is a subset of Telegram service used for setting up user telegram notifications
@@ -87,14 +89,14 @@ func (s *private) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 	comment.Sanitize()
 
-	// check if images are valid
+	// check if images are valid, omit proxied images as they are lazy-loaded
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		err := s.imageService.ResetCleanupTimer(id)
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't renew staged picture cleanup timer", rest.ErrImgNotFound)
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
 			return
 		}
 	}
@@ -127,10 +129,10 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 
-	// check if images are valid
+	// check if images are valid, omit proxied images as they are lazy-loaded
-	for _, id := range s.imageService.ExtractPictures(comment.Text) {
+	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
 		_, err := s.imageService.Load(id)
 		if err != nil {
 			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound)
@@ -150,7 +152,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	id, err := s.dataService.Create(comment)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentRestrictWords)
 		return
 	}
@@ -211,7 +213,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.commentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text, s.disableFancyTextFormatting),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
@@ -219,7 +221,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res, err := s.dataService.EditComment(locator, id, editReq)
-	if err == service.ErrRestrictedWordsFound {
+	if errors.Is(err, service.ErrRestrictedWordsFound) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
@@ -244,7 +246,7 @@ func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
 		}
-		if len(email) > 0 {
+		if email != "" {
 			user.EmailSubscription = true
 		}
 	}
@@ -430,7 +432,7 @@ func (s *private) telegramSubscribeCtrl(w http.ResponseWriter, r *http.Request)
 	var address, siteID string
 	address, siteID, err := s.telegramService.CheckToken(queryToken, user.ID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set telegram for user", rest.ErrInternal)
+		rest.SendErrorJSON(w, r, http.StatusNotFound, err, "request is not verified yet", rest.ErrInternal)
 		return
 	}
 

backend/app/rest/api/rest_private_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
@@ -24,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark42/backend/app/notify"
+	"github.com/umputun/remark42/backend/app/rest/proxy"
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/image"
 )
@@ -69,7 +71,7 @@ func TestRest_CreateFilteredCode(t *testing.T) {
 
 	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
-	assert.NoError(t, err)
+	require.NoError(t, err, string(b))
 	loc := c["locator"].(map[string]interface{})
 	assert.Equal(t, "remark42", loc["site"])
 	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
@@ -79,6 +81,86 @@ func TestRest_CreateFilteredCode(t *testing.T) {
 	assert.True(t, len(c["id"].(string)) > 8)
 }
 
+// based on issue https://github.com/umputun/remark42/issues/1631
+func TestRest_CreateAndPreviewWithImage(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	ts.Close()
+	defer teardown()
+
+	srv.ImageService.ProxyAPI = srv.RemarkURL + "/api/v1/img"
+	srv.ImageProxy = &proxy.Image{
+		HTTP2HTTPS:    true,
+		CacheExternal: true,
+		RoutePath:     "/api/v1/img",
+		RemarkURL:     srv.RemarkURL,
+		ImageService:  srv.ImageService,
+	}
+	srv.CommentFormatter = store.NewCommentFormatter(srv.ImageProxy)
+	// need to recreate the server with new ImageProxy, otherwise old one will be used
+	ts = httptest.NewServer(srv.routes())
+	defer ts.Close()
+
+	var pngRead bool
+	// server with the test PNG image
+	pngServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, e := io.Copy(w, gopherPNG())
+		assert.NoError(t, e)
+		pngRead = true
+	}))
+	defer pngServer.Close()
+
+	t.Run("create", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/comment",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		c := R.JSON{}
+		err = json.Unmarshal(b, &c)
+		require.NoError(t, err, string(b))
+		assert.NotContains(t, c["text"], pngServer.URL)
+		assert.Contains(t, c["text"], srv.RemarkURL)
+		loc := c["locator"].(map[string]interface{})
+		assert.Equal(t, "remark42", loc["site"])
+		assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
+		assert.True(t, len(c["id"].(string)) > 8)
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+	})
+
+	t.Run("preview", func(t *testing.T) {
+		resp, err := post(t, ts.URL+"/api/v1/preview",
+			`{"text": "![](`+pngServer.URL+`/gopher.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+		assert.NoError(t, err)
+		b, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+
+		assert.NotContains(t, string(b), pngServer.URL)
+		assert.Contains(t, string(b), srv.RemarkURL)
+
+		assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
+		// retrieve the image from the cache
+		imgURL := strings.Split(strings.Split(string(b), "src=\"")[1], "\"")[0]
+		// replace srv.RemarkURL with ts.URL
+		imgURL = strings.ReplaceAll(imgURL, srv.RemarkURL, ts.URL)
+		resp, err = http.Get(imgURL)
+		assert.NoError(t, err)
+		b, err = io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode, string(b))
+		assert.NoError(t, resp.Body.Close())
+		// compare image to original gopher png after decoding from base64
+		assert.Equal(t, gopher, base64.StdEncoding.EncodeToString(b))
+
+		assert.Equal(t, true, pngRead, "original image accessed to be shown to user")
+	})
+
+}
+
 func TestRest_CreateOldPost(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -166,6 +248,25 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
 	assert.Equal(t, "invalid comment", c["details"])
 }
 
+func TestRest_CreateRelativeURL(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// check that it's not possible to click insert URL button and not alter the URL in it (which is `url` by default)
+	relativeURLText := `{"text": "here is a link with relative URL: [google.com](url)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
+	resp, err := post(t, ts.URL+"/api/v1/comment", relativeURLText)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	assert.Equal(t, "links should start with mailto:, http:// or https://", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_CreateRejected(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -267,6 +368,56 @@ func TestRest_CreateAndGet(t *testing.T) {
 	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false, IP: ""}, comment.User, "no ip")
 }
 
+func TestRest_CreateWithQuotes(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// create comment with quotes with smartypants
+	resp, err := post(t, ts.URL+"/api/v1/comment",
+		`{"text": "smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id := c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment := store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>smartpants «quoted» text</p>\n", comment.Text)
+	assert.Equal(t, "smartpants \"quoted\" text", comment.Orig)
+
+	// create comment with quotes without smartypants
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/comment",
+		`{"text": "no_smartpants \"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	c = R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.NoError(t, err)
+	id = c["id"].(string)
+
+	// get created comment by id as non-admin
+	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, http.StatusOK, code)
+	comment = store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.NoError(t, err)
+	assert.Equal(t, "<p>no_smartpants &#34;quoted&#34; text</p>\n", comment.Text)
+	assert.Equal(t, "no_smartpants \"quoted\" text", comment.Orig)
+}
+
 func TestRest_Update(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -574,7 +725,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=remark42&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), http.NoBody)
 		assert.NoError(t, err)
-		req.Header.Add("X-JWT", devToken)
+		req.Header.Add("X-JWT", dev2Token)
 		resp, err := client.Do(req)
 		assert.NoError(t, err)
 		assert.NoError(t, resp.Body.Close())
@@ -583,7 +734,7 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, http.StatusOK, vote(1), "first vote allowed")
 	assert.Equal(t, http.StatusBadRequest, vote(1), "second vote rejected")
-	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
@@ -594,7 +745,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -603,7 +754,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, 0, cr.Vote)
 
 	assert.Equal(t, http.StatusOK, vote(-1), "opposite vote allowed one more time")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -612,7 +763,7 @@ func TestRest_Vote(t *testing.T) {
 	assert.Equal(t, -1, cr.Vote)
 
 	assert.Equal(t, http.StatusBadRequest, vote(-1), "dbl vote not allowed")
-	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDev2Auth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, http.StatusOK, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
@@ -709,7 +860,7 @@ func TestRest_EmailAndTelegram(t *testing.T) {
 
 	// issue good token
 	claims := token.Claims{
-		Handshake: &token.Handshake{ID: "dev::good@example.com"},
+		Handshake: &token.Handshake{ID: "provider1_dev::good@example.com"},
 		StandardClaims: jwt.StandardClaims{
 			Audience:  "remark42",
 			ExpiresAt: time.Now().Add(10 * time.Minute).Unix(),
@@ -808,7 +959,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -882,7 +1033,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	var clearUser store.User
 	err = json.Unmarshal(body, &clearUser)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", EmailSubscription: false,
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: false,
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, clearUser)
 
 	// verify email
@@ -916,7 +1067,7 @@ func TestRest_EmailNotification(t *testing.T) {
 	var subscribedUser store.User
 	err = json.Unmarshal(body, &subscribedUser)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", EmailSubscription: true,
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", EmailSubscription: true,
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, subscribedUser)
 
 	// create child comment from another user, email notification expected
@@ -1058,7 +1209,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	// create new comment from dev user
 	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
 		`{"text": "test 123",
-"user": {"name": "dev::good@example.com"},
+"user": {"name": "provider1_dev::good@example.com"},
 "locator":{"url": "https://radio-t.com/blah1",
 "site": "remark42"}}`))
 	assert.NoError(t, err)
@@ -1137,8 +1288,8 @@ func TestRest_TelegramNotification(t *testing.T) {
 	body, err = io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
-	require.Equal(t, http.StatusInternalServerError, resp.StatusCode, string(body))
+	require.Equal(t, http.StatusNotFound, resp.StatusCode, string(body))
-	require.Equal(t, `{"code":0,"details":"can't set telegram for user","error":"not verified"}`+"\n", string(body))
+	require.Equal(t, `{"code":0,"details":"request is not verified yet","error":"not verified"}`+"\n", string(body))
 
 	mockTlgrm.notVerified = false
 
@@ -1186,7 +1337,7 @@ func TestRest_TelegramNotification(t *testing.T) {
 	var user store.User
 	err = json.Unmarshal(body, &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, user)
 
 	// create child comment from another user, telegram notification expected
@@ -1245,7 +1396,7 @@ func TestRest_UserAllData(t *testing.T) {
 	defer teardown()
 
 	// write 3 comments
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
@@ -1270,13 +1421,13 @@ func TestRest_UserAllData(t *testing.T) {
 	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
 
 	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 	ungzBody, err := io.ReadAll(ungzReader)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	strUungzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUungzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 3, strings.Count(strUungzBody, `"text":`), "3 comments inside")
 
 	parsed := struct {
@@ -1286,7 +1437,7 @@ func TestRest_UserAllData(t *testing.T) {
 
 	err = json.Unmarshal(ungzBody, &parsed)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev",
 		Picture: "http://example.com/pic.png", IP: "127.0.0.1", SiteID: "remark42"}, parsed.Info)
 	assert.Equal(t, 3, len(parsed.Comments))
 
@@ -1302,7 +1453,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	user := store.User{ID: "dev", Name: "user name 1"}
+	user := store.User{ID: "provider1_dev", Name: "user name 1"}
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
 
@@ -1329,7 +1480,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	strUngzBody := string(ungzBody)
 	assert.True(t, strings.HasPrefix(strUngzBody,
-		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"provider1_dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false,"site_id":"remark42"}, "comments":[{`))
 	assert.Equal(t, 51, strings.Count(strUngzBody, `"text":`), "51 comments inside")
 }
 
@@ -1353,12 +1504,12 @@ func TestRest_DeleteMe(t *testing.T) {
 	err = json.Unmarshal(body, &m)
 	assert.NoError(t, err)
 	assert.Equal(t, "remark42", m["site"])
-	assert.Equal(t, "dev", m["user_id"])
+	assert.Equal(t, "provider1_dev", m["user_id"])
 
 	tkn := m["token"]
 	claims, err := srv.Authenticator.TokenService().Parse(tkn)
 	assert.NoError(t, err)
-	assert.Equal(t, "dev", claims.User.ID)
+	assert.Equal(t, "provider1_dev", claims.User.ID)
 	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+tkn, m["link"])
 
 	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), http.NoBody)

backend/app/rest/api/rest_public.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/skip2/go-qrcode"
@@ -50,6 +50,8 @@ type pubStore interface {
 
 // GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
+//
+// When `url` parameter is not set (e.g. request is for site-wide comments), does not return deleted comments.
 func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
@@ -77,22 +79,36 @@ func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
 		comments = s.applyView(comments, view)
+
+		var commentsInfo store.PostInfo
+		if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
+			commentsInfo = info
+		}
+
+		if !since.IsZero() { // if since is set, number of comments can be different from total in the DB
+			commentsInfo.Count = 0
+			for _, c := range comments {
+				if !c.Deleted {
+					commentsInfo.Count++
+				}
+			}
+		}
+
+		// post might be readonly without any comments, Info call will fail then and ReadOnly flag should be checked separately
+		if !commentsInfo.ReadOnly && locator.URL != "" && s.dataService.IsReadOnly(locator) {
+			commentsInfo.ReadOnly = true
+		}
+
 		var b []byte
 		switch format {
 		case "tree":
-			tree := service.MakeTree(comments, sort, s.readOnlyAge)
+			withInfo := treeWithInfo{Tree: service.MakeTree(comments, sort), Info: commentsInfo}
-			if tree.Nodes == nil { // eliminate json nil serialization
+			if withInfo.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*service.Node{}
+				withInfo.Nodes = []*service.Node{}
 			}
-			if s.dataService.IsReadOnly(locator) {
+			b, e = encodeJSONWithHTML(withInfo)
-				tree.Info.ReadOnly = true
-			}
-			b, e = encodeJSONWithHTML(tree)
 		default:
-			withInfo := commentsWithInfo{Comments: comments}
+			withInfo := commentsWithInfo{Comments: comments, Info: commentsInfo}
-			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
-				withInfo.Info = info
-			}
 			b, e = encodeJSONWithHTML(withInfo)
 		}
 		return b, e

backend/app/rest/api/rest_public_test.go

@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -76,13 +78,31 @@ func TestRest_Preview(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
+		`{"code":20,"details":"can't load picture from the comment",`+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat",
+			`"error":"can't get image stats for dev_user/bad_picture: stat`,
 	)
 	assert.Contains(t,
 		string(b),
 		"/pics-remark42/staging/dev_user/62/bad_picture: no such file or directory\"}\n",
 	)
+
+	// test quotes with and without smartypants
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>«quoted» text</p>\n", string(b))
+
+	srv.privRest.disableFancyTextFormatting = true
+	resp, err = post(t, ts.URL+"/api/v1/preview", `{"text": "\"quoted\" text", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	b, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, "<p>&#34;quoted&#34; text</p>\n", string(b))
 }
 
 func TestRest_PreviewWithWrongImage(t *testing.T) {
@@ -97,8 +117,8 @@ func TestRest_PreviewWithWrongImage(t *testing.T) {
 	assert.NoError(t, resp.Body.Close())
 	assert.Contains(t,
 		string(b),
-		"{\"code\":20,\"details\":\"can't renew staged picture cleanup timer\","+
+		`{"code":20,"details":"can't load picture from the comment",`+
-			"\"error\":\"can't get image stats for dev_user/bad_picture: stat ",
+			`"error":"can't get image stats for dev_user/bad_picture: stat `,
 	)
 	assert.Contains(t,
 		string(b),
@@ -131,10 +151,10 @@ BKT
 	assert.NoError(t, err)
 	assert.Equal(t,
 		`<h1>h1</h1>
-<pre class="chroma"><code><span class="line"><span class="cl">func TestRest_Preview(t *testing.T) {
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="k">func</span> <span class="n">TestRest_Preview</span><span class="p">(</span><span class="n">t</span> <span class="o">*</span><span class="n">testing</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="p">{</span>
-</span></span><span class="line"><span class="cl">srv, ts := prep(t)
+</span></span><span class="line"><span class="cl"><span class="n">srv</span><span class="p">,</span> <span class="n">ts</span> <span class="p">:</span><span class="o">=</span> <span class="n">prep</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
-</span></span><span class="line"><span class="cl">  require.NotNil(t, srv)
+</span></span><span class="line"><span class="cl">  <span class="n">require</span><span class="o">.</span><span class="n">NotNil</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">srv</span><span class="p">)</span>
-</span></span><span class="line"><span class="cl">}
+</span></span><span class="line"><span class="cl"><span class="p">}</span>
 </span></span></code></pre>`,
 		string(b))
 	assert.NoError(t, resp.Body.Close())
@@ -209,7 +229,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -235,7 +255,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.NoError(t, err)
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
@@ -278,7 +298,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
 
-	tree := service.Tree{}
+	tree := treeWithInfo{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -286,7 +306,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = service.Tree{}
+	tree = treeWithInfo{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, http.StatusOK, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -325,8 +345,8 @@ func TestRest_FindUserView(t *testing.T) {
 	require.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
-	assert.Equal(t, "dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[0].User.ID)
-	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "provider1_dev", comments.Comments[1].User.ID)
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.Equal(t, "", comments.Comments[1].Text)
 
@@ -440,7 +460,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code, "noting for user blah")
 	assert.Equal(t, `{"comments":[],"count":0}`+"\n", comments)
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -459,7 +479,7 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 
 	{
-		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=dev&skip=1&limit=2")
+		res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev&skip=1&limit=2")
 		assert.Equal(t, http.StatusOK, code)
 
 		resp := struct {
@@ -477,6 +497,203 @@ func TestRest_FindUserComments(t *testing.T) {
 	}
 }
 
+func TestRest_FindUserComments_CWE_918(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second}, []string{"radio-t.com"}) // required for extracting the title, bad URL test
+	defer srv.DataService.TitleExtractor.Close()
+	defer teardown()
+
+	backendRequestedArbitraryServer := false
+	arbitraryServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		t.Logf("request received: %+v", r)
+		backendRequestedArbitraryServer = true
+	}))
+	defer arbitraryServer.Close()
+
+	arbitraryURLComment := store.Comment{Text: "arbitrary URL request test",
+		Locator: store.Locator{SiteID: "remark42", URL: arbitraryServer.URL}}
+
+	assert.False(t, backendRequestedArbitraryServer)
+	addComment(t, arbitraryURLComment, ts)
+	assert.False(t, backendRequestedArbitraryServer,
+		"no request is expected to the test server as it's not in the list of the allowed domains for the title extractor")
+
+	res, code := get(t, ts.URL+"/api/v1/comments?site=remark42&user=provider1_dev")
+	assert.Equal(t, http.StatusOK, code)
+
+	resp := struct {
+		Comments []store.Comment
+		Count    int
+	}{}
+
+	err := json.Unmarshal([]byte(res), &resp)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(resp.Comments), "should have 2 comments")
+
+	assert.Equal(t, "", resp.Comments[0].PostTitle, "empty from the first post")
+	assert.Equal(t, arbitraryServer.URL, resp.Comments[0].Locator.URL, "arbitrary URL provided by the request")
+}
+
+func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
+	// test that comment counting is consistent between tree and plain formats
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	commentLocator := store.Locator{URL: "test-url", SiteID: "remark42"}
+
+	// vote for comment multiple times
+	setScore := func(locator store.Locator, id string, val int) {
+		abs := func(x int) int {
+			if x < 0 {
+				return -x
+			}
+			return x
+		}
+		for i := 0; i < abs(val); i++ {
+			_, err := srv.DataService.Vote(service.VoteReq{
+				Locator:   locator,
+				CommentID: id,
+				// unique user ID is needed for correct counting of controversial votes
+				UserID: "user" + strconv.Itoa(val) + strconv.Itoa(i),
+				Val:    val > 0,
+			})
+			require.NoError(t, err)
+		}
+	}
+
+	// Adding initial comments (8 to test-url and 1 to another-url) and voting, and delete two of comments to the first post.
+	// With sleep so that at least few millisecond pass between each comment
+	// and later we would be able to use that in "since" filter with millisecond precision
+	ids := make([]string, 9)
+	timestamps := make([]time.Time, 9)
+	c1 := store.Comment{Text: "top-level comment 1", Locator: commentLocator}
+	ids[0], timestamps[0] = addCommentGetCreatedTime(t, c1, ts)
+	// #3 by score
+	setScore(commentLocator, ids[0], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c2 := store.Comment{Text: "top-level comment 2", Locator: commentLocator}
+	ids[1], timestamps[1] = addCommentGetCreatedTime(t, c2, ts)
+	// #2 by score
+	setScore(commentLocator, ids[1], 2)
+	time.Sleep(time.Millisecond * 5)
+
+	c3 := store.Comment{Text: "second-level comment 1", ParentID: ids[0], Locator: commentLocator}
+	ids[2], timestamps[2] = addCommentGetCreatedTime(t, c3, ts)
+	// #1 by score
+	setScore(commentLocator, ids[2], 10)
+	time.Sleep(time.Millisecond * 5)
+
+	c4 := store.Comment{Text: "third-level comment 1", ParentID: ids[2], Locator: commentLocator}
+	ids[3], timestamps[3] = addCommentGetCreatedTime(t, c4, ts)
+	// #5 by score, #1 by controversy
+	setScore(commentLocator, ids[3], 4)
+	setScore(commentLocator, ids[3], -4)
+	time.Sleep(time.Millisecond * 5)
+
+	c5 := store.Comment{Text: "second-level comment 2", ParentID: ids[1], Locator: commentLocator}
+	ids[4], timestamps[4] = addCommentGetCreatedTime(t, c5, ts)
+	// #5 by score, #2 by controversy
+	setScore(commentLocator, ids[4], 2)
+	setScore(commentLocator, ids[4], -3)
+	time.Sleep(time.Millisecond * 5)
+
+	c6 := store.Comment{Text: "third-level comment 2", ParentID: ids[4], Locator: commentLocator}
+	ids[5], timestamps[5] = addCommentGetCreatedTime(t, c6, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[5], 10)
+	setScore(commentLocator, ids[5], -10)
+	time.Sleep(time.Millisecond * 5)
+
+	c7 := store.Comment{Text: "top-level comment 3", Locator: commentLocator}
+	ids[6], timestamps[6] = addCommentGetCreatedTime(t, c7, ts)
+	// #6 by score, #4 by controversy
+	setScore(commentLocator, ids[6], -3)
+	setScore(commentLocator, ids[6], 1)
+	time.Sleep(time.Millisecond * 5)
+
+	c8 := store.Comment{Text: "second-level comment 3", ParentID: ids[6], Locator: commentLocator}
+	ids[7], timestamps[7] = addCommentGetCreatedTime(t, c8, ts)
+	// deleted later so not visible in site-wide requests
+	setScore(commentLocator, ids[7], -20)
+
+	c9 := store.Comment{Text: "comment to post 2", Locator: store.Locator{URL: "another-url", SiteID: "remark42"}}
+	ids[8], timestamps[8] = addCommentGetCreatedTime(t, c9, ts)
+	// #7 by score
+	setScore(store.Locator{URL: "another-url", SiteID: "remark42"}, ids[8], -25)
+
+	// delete two comments bringing the total from 9 to 6
+	err := srv.DataService.Delete(commentLocator, ids[7], store.SoftDelete)
+	assert.NoError(t, err)
+	err = srv.DataService.Delete(commentLocator, ids[5], store.HardDelete)
+	assert.NoError(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
+
+	commentLocator.URL = "readonly-test"
+	// set post without comments to read-only
+	assert.NoError(t, srv.DataService.SetReadOnly(commentLocator, true))
+
+	sinceTenSecondsAgo := strconv.FormatInt(time.Now().Add(-time.Second*10).UnixNano()/1000000, 10)
+	sinceTS := make([]string, 9)
+	formattedTS := make([]string, 9)
+	for i, created := range timestamps {
+		sinceTS[i] = strconv.FormatInt(created.UnixNano()/1000000, 10)
+		formattedTS[i] = created.Format(time.RFC3339Nano)
+	}
+	t.Logf("last timestamp: %v", timestamps[7])
+
+	testCases := []struct {
+		params       string
+		expectedBody string
+	}{
+		{"", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=plain", fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"format=plain&url=test-url", fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTenSecondsAgo, fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[0], fmt.Sprintf(`"info":{"count":7,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[0], fmt.Sprintf(`"info":{"url":"test-url","count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[1], fmt.Sprintf(`"info":{"count":6,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[1], fmt.Sprintf(`"info":{"url":"test-url","count":5,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"since=" + sinceTS[4], fmt.Sprintf(`"info":{"count":3,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[8])},
+		{"url=test-url&since=" + sinceTS[4], fmt.Sprintf(`"info":{"url":"test-url","count":2,"first_time":%q,"last_time":%q}`, formattedTS[0], formattedTS[7])},
+		{"format=tree", `"info":{"count":7`},
+		{"format=tree&url=test-url", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=+time", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=+time", `"info":{"url":"test-url","count":6`},
+		{"format=tree&sort=-score", `"info":{"count":7`},
+		{"format=tree&url=test-url&sort=-score", `"info":{"url":"test-url","count":6`},
+		{"sort=+time", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-time", fmt.Sprintf(`"score":1,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[0])},
+		{"sort=+score", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[2])},
+		{"sort=+score&url=test-url", fmt.Sprintf(`"score":10,"vote":0,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[2])},
+		{"sort=-score", fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":7`, formattedTS[8])},
+		{"sort=-score&url=test-url", fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":6`, formattedTS[6])},
+		{"sort=-time&since=" + sinceTS[4], fmt.Sprintf(`"score":-1,"vote":0,"controversy":2.924017738212866,"time":%q}],"info":{"count":3`, formattedTS[4])},
+		{"sort=-score&since=" + sinceTS[3], fmt.Sprintf(`"score":-25,"vote":0,"time":%q}],"info":{"count":4`, formattedTS[8])},
+		{"sort=-score&url=test-url&since=" + sinceTS[3], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":3`, formattedTS[6])},
+		{"sort=+controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":-2,"vote":0,"controversy":1.5874010519681994,"time":%q}],"info":{"url":"test-url","count":1`, formattedTS[6])},
+		// three comments of which last one deleted and doesn't have controversy so returned last
+		{"sort=-controversy&url=test-url&since=" + sinceTS[5], fmt.Sprintf(`"score":0,"vote":0,"time":%q,"delete":true}],"info":{"url":"test-url","count":1`, formattedTS[7])},
+		// test readonly status for the post without comments
+		{"url=readonly-test", `"info":{"count":0,"read_only":true`},
+		{"format=tree&url=readonly-test", `"info":{"count":0,"read_only":true`},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.params, func(t *testing.T) {
+			url := fmt.Sprintf(ts.URL+"/api/v1/find?site=remark42&%s", tc.params)
+			body, code := get(t, url)
+			assert.Equal(t, http.StatusOK, code)
+			assert.Contains(t, body, tc.expectedBody)
+			t.Log(body)
+			// prevent hit limiter from engaging
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
 func TestRest_UserInfo(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -486,7 +703,7 @@ func TestRest_UserInfo(t *testing.T) {
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
 	assert.NoError(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png",
+	assert.Equal(t, store.User{Name: "developer one", ID: "provider1_dev", Picture: "http://example.com/pic.png",
 		IP: "127.0.0.1", SiteID: "remark42"}, user)
 }
 

backend/app/rest/api/rest_test.go

@@ -18,8 +18,9 @@ import (
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/token"
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,13 +41,15 @@ import (
 // To generate a token, enter one of the tokens here into https://jwt.io, change the secret to one you're using in your test
 // ("secret" in case of startupT), and alter the fields you want to be changed.
 
-var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
+var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.dirTS_ahSF6375sdO2iodm2K2UmRTzQNQMFiHuTQCVs`
+
+var dev2Token = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6InByb3ZpZGVyMV9kZXYyIiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.qsR_PupfjIq7uw0eAuyGV8nsUoMx9v541c9olnRInRQ`
 
 var anonToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImFub255bW91cyB0ZXN0IHVzZXIiLCJpZCI6ImFub255bW91c190ZXN0X3VzZXIiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoiYW5vbkBleGFtcGxlLmNvbSJ9fQ.gAae2WMxZNZE5ebVboptPEyQ7Nk6EQxciNnGJ_mPOuU`
 
 var emailUserToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6Imdvb2RAZXhhbXBsZS5jb20gdGVzdCB1c2VyIiwiaWQiOiJlbWFpbF9mNWRmZTlkMmU2YmQ3NWZjNzRlYTVmYWJmMjczYjQ1YjViYWViMTk1IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Imdvb2RAZXhhbXBsZS5jb20ifX0.vH2HN1JpuXL8okTJq1A-zGHQ-l2ILcwxvDDEmu2zwks`
 
-var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiaHR0cDovL2V4YW1wbGUuY29tL3BpYy5wbmciLCJpcCI6IjEyNy4wLjAuMSIsImVtYWlsIjoibWVAZXhhbXBsZS5jb20ifX0.FuTTocVtcxr4VjpfIICvU2yOb3su28VkDzj94H9Q3xY`
+var devTokenBadAud = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0Ml9iYWQiLCJleHAiOjM3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTIxODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJwcm92aWRlcjFfZGV2IiwicGljdHVyZSI6Imh0dHA6Ly9leGFtcGxlLmNvbS9waWMucG5nIiwiaXAiOiIxMjcuMC4wLjEiLCJlbWFpbCI6Im1lQGV4YW1wbGUuY29tIn19.X-lvnHvBz6VfEbVV4f-bjcZuLY5pYtvEansk_TQMrX8`
 
 var adminUmputunToken = `eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6MTk1NDU5Nzk4MCwianRpIjoiOTdhMmUwYWM0ZGM3ZDVmNjkyNmQ1ZTg2MjBhY2VmOWE0MGMwIiwiaWF0IjoxNDU0NTk3NjgwLCJpc3MiOiJyZW1hcms0MiIsInVzZXIiOnsibmFtZSI6IlVtcHV0dW4iLCJpZCI6ImdpdGh1Yl9lZjBmNzA2YTciLCJwaWN0dXJlIjoiaHR0cHM6Ly9yZW1hcms0Mi5yYWRpby10LmNvbS9hcGkvdjEvYXZhdGFyL2NiNDJmZjQ5M2FkZTY5NmQ4OGEzYTU5MGYxMzZhZTllMzRkZTdjMWIuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.dZiOjWHguo9f42XCMooMcv4EmYFzifl_-LEvPZHCtks`
 
@@ -125,7 +128,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -175,7 +178,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}
@@ -191,7 +194,7 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 }
 
 func TestRest_rejectAnonUser(t *testing.T) {
-	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		fmt.Fprintln(w, "Hello")
 	}))))
 	defer ts.Close()
@@ -301,10 +304,10 @@ func TestRest_cacheControl(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", tt.url, nil)
+			req := httptest.NewRequest("GET", tt.url, http.NoBody)
 			w := httptest.NewRecorder()
 
-			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := cacheControl(tt.exp, tt.version)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -329,10 +332,10 @@ func TestRest_frameAncestors(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req := httptest.NewRequest("GET", "http://example.com", http.NoBody)
 			w := httptest.NewRecorder()
 
-			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := frameAncestors(tt.hosts)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -363,12 +366,12 @@ func TestRest_subscribersOnly(t *testing.T) {
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req := httptest.NewRequest("GET", "http://example.com", http.NoBody)
 			if tt.setUser {
 				req = token.SetUserInfo(req, tt.user)
 			}
 			w := httptest.NewRecorder()
-			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := subscribersOnly(tt.subsOnly)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, tt.status, resp.StatusCode)
@@ -400,7 +403,7 @@ func Test_validEmailAuth(t *testing.T) {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			req := httptest.NewRequest("GET", "http://example.com"+tt.req, http.NoBody)
 			w := httptest.NewRecorder()
-			h := validEmailAuth()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+			h := validEmailAuth()(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
 			h.ServeHTTP(w, req)
 			resp := w.Result()
 			assert.Equal(t, tt.status, resp.StatusCode)
@@ -435,7 +438,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "remark42"})
 	require.NoError(t, err)
 
-	memCache := cache.NewScache(cache.NewNopCache())
+	memCache := cache.NewScache[[]byte](cache.NewNopCache[[]byte]())
 
 	astore := adminstore.NewStaticStore("123456", []string{"remark42"}, []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
@@ -455,7 +458,7 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 		DataService: dataStore,
 		Authenticator: auth.NewService(auth.Opts{
 			AdminPasswd:  "password",
-			SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
+			SecretReader: token.SecretFunc(func(string) (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS(tmp + "/ava-remark42"),
 		}),
 		Cache:     memCache,
@@ -484,16 +487,26 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
 			Cache:             memCache,
 			KeyStore:          astore,
 		},
-		NotifyService: notify.NopService,
+		NotifyService:    notify.NopService,
-		EmojiEnabled:  true,
+		EmojiEnabled:     true,
+		openRouteLimiter: 100,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
+	// add some providers. Needed because we don't allow users with unlisted providers to authenticate
+	providers := []string{"provider1", "anonymous", "github", "email"}
+	for _, p := range providers {
+		srv.Authenticator.AddDirectProvider(p, provider.CredCheckerFunc(func(_, _ string) (ok bool, err error) {
+			return true, nil
+		}))
+	}
+
 	for _, h := range srvHook {
 		h(srv)
 	}
 
-	ts = httptest.NewServer(srv.routes())
+	routes := srv.routes()
+	ts = httptest.NewServer(routes)
 
 	teardown = func() {
 		ts.Close()
@@ -552,6 +565,20 @@ func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	return string(b), r.StatusCode
 }
 
+func getWithDev2Auth(t *testing.T, url string) (body string, code int) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	defer client.CloseIdleConnections()
+	req, err := http.NewRequest("GET", url, http.NoBody)
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", dev2Token)
+	r, err := client.Do(req)
+	require.NoError(t, err)
+	b, err := io.ReadAll(r.Body)
+	assert.NoError(t, err)
+	require.NoError(t, r.Body.Close())
+	return string(b), r.StatusCode
+}
+
 func getWithAdminAuth(t *testing.T, url string) (response string, statusCode int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	defer client.CloseIdleConnections()
@@ -574,7 +601,7 @@ func post(t *testing.T, url, body string) (*http.Response, error) {
 	return client.Do(req)
 }
 
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+func addCommentGetCreatedTime(t *testing.T, c store.Comment, ts *httptest.Server) (id string, created time.Time) {
 	b, err := json.Marshal(c)
 	require.NoError(t, err, "can't marshal comment %+v", c)
 
@@ -594,7 +621,14 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	err = json.Unmarshal(b, &crResp)
 	require.NoError(t, err)
 	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
+	created, err = time.Parse(time.RFC3339, crResp["time"].(string))
+	require.NoError(t, err)
+	return crResp["id"].(string), created
+}
+
+func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+	id, _ := addCommentGetCreatedTime(t, c, ts)
+	return id
 }
 
 func requireAdminOnly(t *testing.T, req *http.Request) {
@@ -633,5 +667,9 @@ func waitForHTTPSServerStart(port int) {
 }
 
 func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
+	goleak.VerifyTestMain(
+		m,
+		// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
+		goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
+	)
 }

backend/app/rest/api/rss.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"time"
 
-	cache "github.com/go-pkgz/lcw"
+	cache "github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"github.com/gorilla/feeds"
 
@@ -56,7 +56,6 @@ func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-
 	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}

backend/app/rest/api/ssl_test.go

@@ -21,7 +21,7 @@ func TestSSL_Redirect(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 
@@ -56,7 +56,7 @@ func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
 
 	client := http.Client{
 		// prevent http redirect
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		CheckRedirect: func(*http.Request, []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}

backend/app/rest/api/testdata/commento.json

@@ -0,0 +1,116 @@
+{
+  "version": 1,
+  "comments": [
+    {
+      "commentHex": "e7a2ef4b4aa1414a7ee65a989889aaecd9d5e7e3bca598ea7a967b4dbcaa8e11",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2022-10-25T07:25:46.807555Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "a29e741145daceb4ca5b3e5e279e05b56f73c04703d93b944718ef757e15317f",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-07-26T12:24:55.058552Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-31T11:03:25.403282Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "6d3bb64ff73b5f9d6a959212ffde472a51abf8bdefaa5ed843659796bceef9de",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:23:47.112062Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-23T12:33:03.370182Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "d0ad6f11cf0c5f8e17457a378a6bb789f412c6b7ef7ada4ae06ec8451f7a18aa",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "root",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-10-18T01:18:38.193625Z",
+      "direction": 0,
+      "deleted": false
+    },
+    {
+      "commentHex": "098960fd01c1fc7c0d3ea428f52fab97ea5c18aa52f3565bba679224daddc687",
+      "domain": "example.com",
+      "url": "/example",
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "markdown": "",
+      "html": "",
+      "parentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
+      "score": 0,
+      "state": "approved",
+      "creationDate": "2023-11-01T22:24:04.639965Z",
+      "direction": 0,
+      "deleted": false
+    }
+  ],
+  "commenters": [
+    {
+      "commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
+      "email": "undefined",
+      "name": "blank",
+      "link": "undefined",
+      "photo": "undefined",
+      "provider": "anon",
+      "joinDate": "2022-06-09T15:54:29.865919Z",
+      "isModerator": false,
+      "deleted": false
+    }
+  ]
+}

backend/app/rest/proxy/image.go

@@ -57,7 +57,7 @@ func (p Image) extract(commentHTML string, imgSrcPred func(string) bool) ([]stri
 		return nil, fmt.Errorf("can't create document: %w", err)
 	}
 	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		if im, ok := s.Attr("src"); ok {
 			if imgSrcPred(im) {
 				result = append(result, im)
@@ -98,6 +98,10 @@ func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
 	if img == nil {
 		img, err = p.downloadImage(context.Background(), imgURL)
 		if err != nil {
+			if strings.Contains(err.Error(), "invalid content type") {
+				rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid content type", rest.ErrImgNotFound)
+				return
+			}
 			rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't get image "+imgURL, rest.ErrAssetNotFound)
 			return
 		}
@@ -159,12 +163,17 @@ func (p Image) downloadImage(ctx context.Context, imgURL string) ([]byte, error)
 	if err != nil {
 		return nil, fmt.Errorf("can't download image %s: %w", imgURL, err)
 	}
-	defer resp.Body.Close()
+	defer resp.Body.Close() //nolint gosec // we don't care about response body
 
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("got unsuccessful response status %d while fetching %s", resp.StatusCode, imgURL)
 	}
 
+	contentType := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(contentType, "image/") {
+		return nil, fmt.Errorf("invalid content type %s", contentType)
+	}
+
 	imgData, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read image body")

backend/app/rest/proxy/image_test.go

@@ -95,7 +95,7 @@ func TestImage_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	// no image supposed to be cached
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",
@@ -108,31 +108,45 @@ func TestImage_Routes(t *testing.T) {
 	httpSrv := imgHTTPTestsServer(t)
 	defer httpSrv.Close()
 
-	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+	t.Run("valid image", func(t *testing.T) {
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+		require.NoError(t, err)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+		assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+	})
 
-	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	t.Run("no image", func(t *testing.T) {
-	require.NoError(t, err)
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
-	assert.NoError(t, resp.Body.Close())
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
+		require.NoError(t, err)
-	assert.Equal(t, "1462", resp.Header["Content-Length"][0])
+		assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+	})
 
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
+	t.Run("bad encoding", func(t *testing.T) {
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
-	require.NoError(t, err)
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	assert.NoError(t, resp.Body.Close())
+		require.NoError(t, err)
-	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+		assert.NoError(t, resp.Body.Close())
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 2, len(imageStore.LoadCalls()))
+	})
 
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
+	t.Run("non-image reference", func(t *testing.T) {
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
+		encodedImgURL := base64.URLEncoding.EncodeToString([]byte("https://google.com"))
-	require.NoError(t, err)
+		resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	assert.NoError(t, resp.Body.Close())
+		require.NoError(t, err)
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.NoError(t, resp.Body.Close())
-	assert.Equal(t, 2, len(imageStore.LoadCalls()))
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, 3, len(imageStore.LoadCalls()))
+	})
 }
 
 func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		RemarkURL:    "https://demo.remark42.com",
 		RoutePath:    "/api/v1/proxy",
@@ -158,10 +172,10 @@ func TestImage_DisabledCachingAndHTTP2HTTPS(t *testing.T) {
 
 func TestImage_RoutesCachingImage(t *testing.T) {
 	imageStore := image.StoreMock{
-		LoadFunc: func(id string) ([]byte, error) {
+		LoadFunc: func(string) ([]byte, error) {
 			return nil, nil
 		},
-		SaveFunc: func(id string, img []byte) error {
+		SaveFunc: func(string, []byte) error {
 			return nil
 		},
 	}
@@ -196,7 +210,7 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 func TestImage_RoutesUsingCachedImage(t *testing.T) {
 	// In order to validate that cached data used cache "will return" some other data from what http server would
 	testImage := []byte(fmt.Sprintf("%256s", "X"))
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) {
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) {
 		return testImage, nil
 	}}
 	img := Image{
@@ -226,7 +240,7 @@ func TestImage_RoutesUsingCachedImage(t *testing.T) {
 
 func TestImage_RoutesTimedOut(t *testing.T) {
 	// no image supposed to be cached
-	imageStore := image.StoreMock{LoadFunc: func(id string) ([]byte, error) { return nil, nil }}
+	imageStore := image.StoreMock{LoadFunc: func(string) ([]byte, error) { return nil, nil }}
 	img := Image{
 		HTTP2HTTPS:   true,
 		RemarkURL:    "https://demo.remark42.com",

backend/app/store/admin/admin.go

@@ -8,6 +8,9 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
+//go:generate moq --out admin_mock.go . Store
+
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key(siteID string) (key string, err error)

backend/app/store/admin/admin_mock.go

@@ -0,0 +1,256 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package admin
+
+import (
+	"sync"
+)
+
+// Ensure, that StoreMock does implement Store.
+// If this is not the case, regenerate this file with moq.
+var _ Store = &StoreMock{}
+
+// StoreMock is a mock implementation of Store.
+//
+//	func TestSomethingThatUsesStore(t *testing.T) {
+//
+//		// make and configure a mocked Store
+//		mockedStore := &StoreMock{
+//			AdminsFunc: func(siteID string) ([]string, error) {
+//				panic("mock out the Admins method")
+//			},
+//			EmailFunc: func(siteID string) (string, error) {
+//				panic("mock out the Email method")
+//			},
+//			EnabledFunc: func(siteID string) (bool, error) {
+//				panic("mock out the Enabled method")
+//			},
+//			KeyFunc: func(siteID string) (string, error) {
+//				panic("mock out the Key method")
+//			},
+//			OnEventFunc: func(siteID string, et EventType) error {
+//				panic("mock out the OnEvent method")
+//			},
+//		}
+//
+//		// use mockedStore in code that requires Store
+//		// and then make assertions.
+//
+//	}
+type StoreMock struct {
+	// AdminsFunc mocks the Admins method.
+	AdminsFunc func(siteID string) ([]string, error)
+
+	// EmailFunc mocks the Email method.
+	EmailFunc func(siteID string) (string, error)
+
+	// EnabledFunc mocks the Enabled method.
+	EnabledFunc func(siteID string) (bool, error)
+
+	// KeyFunc mocks the Key method.
+	KeyFunc func(siteID string) (string, error)
+
+	// OnEventFunc mocks the OnEvent method.
+	OnEventFunc func(siteID string, et EventType) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Admins holds details about calls to the Admins method.
+		Admins []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Email holds details about calls to the Email method.
+		Email []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Enabled holds details about calls to the Enabled method.
+		Enabled []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// Key holds details about calls to the Key method.
+		Key []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+		}
+		// OnEvent holds details about calls to the OnEvent method.
+		OnEvent []struct {
+			// SiteID is the siteID argument value.
+			SiteID string
+			// Et is the et argument value.
+			Et EventType
+		}
+	}
+	lockAdmins  sync.RWMutex
+	lockEmail   sync.RWMutex
+	lockEnabled sync.RWMutex
+	lockKey     sync.RWMutex
+	lockOnEvent sync.RWMutex
+}
+
+// Admins calls AdminsFunc.
+func (mock *StoreMock) Admins(siteID string) ([]string, error) {
+	if mock.AdminsFunc == nil {
+		panic("StoreMock.AdminsFunc: method is nil but Store.Admins was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockAdmins.Lock()
+	mock.calls.Admins = append(mock.calls.Admins, callInfo)
+	mock.lockAdmins.Unlock()
+	return mock.AdminsFunc(siteID)
+}
+
+// AdminsCalls gets all the calls that were made to Admins.
+// Check the length with:
+//
+//	len(mockedStore.AdminsCalls())
+func (mock *StoreMock) AdminsCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockAdmins.RLock()
+	calls = mock.calls.Admins
+	mock.lockAdmins.RUnlock()
+	return calls
+}
+
+// Email calls EmailFunc.
+func (mock *StoreMock) Email(siteID string) (string, error) {
+	if mock.EmailFunc == nil {
+		panic("StoreMock.EmailFunc: method is nil but Store.Email was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEmail.Lock()
+	mock.calls.Email = append(mock.calls.Email, callInfo)
+	mock.lockEmail.Unlock()
+	return mock.EmailFunc(siteID)
+}
+
+// EmailCalls gets all the calls that were made to Email.
+// Check the length with:
+//
+//	len(mockedStore.EmailCalls())
+func (mock *StoreMock) EmailCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEmail.RLock()
+	calls = mock.calls.Email
+	mock.lockEmail.RUnlock()
+	return calls
+}
+
+// Enabled calls EnabledFunc.
+func (mock *StoreMock) Enabled(siteID string) (bool, error) {
+	if mock.EnabledFunc == nil {
+		panic("StoreMock.EnabledFunc: method is nil but Store.Enabled was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockEnabled.Lock()
+	mock.calls.Enabled = append(mock.calls.Enabled, callInfo)
+	mock.lockEnabled.Unlock()
+	return mock.EnabledFunc(siteID)
+}
+
+// EnabledCalls gets all the calls that were made to Enabled.
+// Check the length with:
+//
+//	len(mockedStore.EnabledCalls())
+func (mock *StoreMock) EnabledCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockEnabled.RLock()
+	calls = mock.calls.Enabled
+	mock.lockEnabled.RUnlock()
+	return calls
+}
+
+// Key calls KeyFunc.
+func (mock *StoreMock) Key(siteID string) (string, error) {
+	if mock.KeyFunc == nil {
+		panic("StoreMock.KeyFunc: method is nil but Store.Key was just called")
+	}
+	callInfo := struct {
+		SiteID string
+	}{
+		SiteID: siteID,
+	}
+	mock.lockKey.Lock()
+	mock.calls.Key = append(mock.calls.Key, callInfo)
+	mock.lockKey.Unlock()
+	return mock.KeyFunc(siteID)
+}
+
+// KeyCalls gets all the calls that were made to Key.
+// Check the length with:
+//
+//	len(mockedStore.KeyCalls())
+func (mock *StoreMock) KeyCalls() []struct {
+	SiteID string
+} {
+	var calls []struct {
+		SiteID string
+	}
+	mock.lockKey.RLock()
+	calls = mock.calls.Key
+	mock.lockKey.RUnlock()
+	return calls
+}
+
+// OnEvent calls OnEventFunc.
+func (mock *StoreMock) OnEvent(siteID string, et EventType) error {
+	if mock.OnEventFunc == nil {
+		panic("StoreMock.OnEventFunc: method is nil but Store.OnEvent was just called")
+	}
+	callInfo := struct {
+		SiteID string
+		Et     EventType
+	}{
+		SiteID: siteID,
+		Et:     et,
+	}
+	mock.lockOnEvent.Lock()
+	mock.calls.OnEvent = append(mock.calls.OnEvent, callInfo)
+	mock.lockOnEvent.Unlock()
+	return mock.OnEventFunc(siteID, et)
+}
+
+// OnEventCalls gets all the calls that were made to OnEvent.
+// Check the length with:
+//
+//	len(mockedStore.OnEventCalls())
+func (mock *StoreMock) OnEventCalls() []struct {
+	SiteID string
+	Et     EventType
+} {
+	var calls []struct {
+		SiteID string
+		Et     EventType
+	}
+	mock.lockOnEvent.RLock()
+	calls = mock.calls.OnEvent
+	mock.lockOnEvent.RUnlock()
+	return calls
+}

backend/app/store/admin/admin_test.go

@@ -6,6 +6,20 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStaticStore_StoreWithoutSites(t *testing.T) {
+	var ks Store = NewStaticKeyStore("key123")
+	enabled, err := ks.Enabled("any")
+	assert.NoError(t, err)
+	assert.True(t, enabled, "on empty store all sites are enabled")
+	assert.NoError(t, ks.OnEvent("test", EvCreate), "static store does nothing OnEvent")
+
+	// empty key
+	ks = NewStaticKeyStore("")
+	key, err := ks.Key("any")
+	assert.Error(t, err, "empty key")
+	assert.Empty(t, key)
+}
+
 func TestStaticStore_Get(t *testing.T) {
 	var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"},
 		[]string{"123", "xyz"}, "aa@example.com")

backend/app/store/comment.go

@@ -45,9 +45,9 @@ type Edit struct {
 
 // PostInfo holds summary for given post url
 type PostInfo struct {
-	URL      string    `json:"url"`
+	URL      string    `json:"url,omitempty"` // can be attached to site-wide comments but won't be set then
 	Count    int       `json:"count"`
-	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"`
+	ReadOnly bool      `json:"read_only,omitempty" bson:"read_only,omitempty"` // can be attached to site-wide comments but won't be set then
 	FirstTS  time.Time `json:"first_time,omitempty" bson:"first_time,omitempty"`
 	LastTS   time.Time `json:"last_time,omitempty" bson:"last_time,omitempty"`
 }
@@ -98,6 +98,7 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
 	c.Text = ""
 	c.Orig = ""
 	c.Score = 0
+	c.Controversy = 0
 	c.Votes = map[string]bool{}
 	c.VotedIPs = make(map[string]VotedIPInfo)
 	c.Edit = nil
@@ -121,8 +122,8 @@ func (c *Comment) Sanitize() {
 	// special case for embedding the quotes from Twitter
 	p.AllowAttrs("class").Matching(regexp.MustCompile("^twitter-tweet$")).OnElements("blockquote")
 	// this is list of <span> tag classes which could be produced by chroma code renderer
-	// source: https://github.com/alecthomas/chroma/blob/cc2dd5b/types.go#L211-L307
+	// source: https://github.com/alecthomas/chroma/blob/c263f6f/types.go#L209-L306
-	const codeSpanClassRegex = "^(bg|chroma|line|ln|lnt|hl|lntable|lntd|cl|w|err|x|k|kc" +
+	const codeSpanClassRegex = "^(bg|chroma|line|ln|lnt|hl|lntable|lntd|lnlinks|cl|w|err|x|k|kc" +
 		"|kd|kn|kp|kr|kt|n|na|nb|bp|nc|no|nd|ni|ne|nf|fm|py|nl|nn|nx|nt|nv|vc|vg" +
 		"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
 		"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
@@ -146,7 +147,7 @@ func (c *Comment) Snippet(limit int) string {
 	if size < limit {
 		return cleanText
 	}
-	snippet := []rune(cleanText)[:size]
+	snippet := []rune(cleanText)[:limit]
 	// go back in snippet and found the first space
 	for i := len(snippet) - 1; i >= 0; i-- {
 		if snippet[i] == ' ' {
@@ -154,6 +155,10 @@ func (c *Comment) Snippet(limit int) string {
 			break
 		}
 	}
+	// Don't add a space if comment is just a one single word which has been truncated.
+	if len(snippet) == limit {
+		return string(snippet) + "..."
+	}
 	return string(snippet) + " ..."
 }
 
@@ -178,8 +183,8 @@ func (c *Comment) escapeHTMLWithSome(inp string) string {
 	return res
 }
 
-// SanitizeText used to sanitize any input string
+// SanitizeText used to sanitize any input string, and removes any HTML tags
 func (c *Comment) SanitizeText(inp string) string {
-	clean := bluemonday.UGCPolicy().Sanitize(inp)
+	clean := bluemonday.StrictPolicy().Sanitize(inp)
-	return c.escapeHTMLWithSome(clean)
+	return strings.TrimSpace(c.escapeHTMLWithSome(clean))
 }

backend/app/store/comment_test.go

@@ -22,7 +22,7 @@ func TestComment_Sanitize(t *testing.T) {
 			},
 			out: Comment{
 				Text: "blah XSS\n\t",
-				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
+				User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name"},
 			},
 		},
 		{
@@ -88,6 +88,14 @@ func TestComment_Sanitize(t *testing.T) {
 			inp: Comment{Text: "blah blah", PostTitle: "<script>alert()</script>something"},
 			out: Comment{Text: "blah blah", PostTitle: "something"},
 		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "<a href=\"https://example.com\">test</a>"},
+			out: Comment{Text: "blah blah", PostTitle: "test"},
+		},
+		{
+			inp: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"}, // link is left as-is, but not rendered as <a>
+			out: Comment{Text: "blah blah", PostTitle: "https://example.com/blah"},
+		},
 		{
 			inp: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw">@marcsheep</a> <a href="https://t.co/ULnVpG8w55">pic.twitter.com/ULnVpG8w55</a></p>&mdash; Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw">November 9, 2020</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>`, PostTitle: "Twitter quote"},
 			out: Comment{Text: `<blockquote class="twitter-tweet"><p lang="es" dir="ltr">Silicon iMac Concept<a href="https://t.co/7ga95QxVXn" rel="nofollow">https://t.co/7ga95QxVXn</a> by <a href="https://twitter.com/marcsheep?ref_src=twsrc%5Etfw" rel="nofollow">@marcsheep</a> <a href="https://t.co/ULnVpG8w55" rel="nofollow">pic.twitter.com/ULnVpG8w55</a></p>— Andreas Storm (@avstorm) <a href="https://twitter.com/avstorm/status/1325693387798933504?ref_src=twsrc%5Etfw" rel="nofollow">November 9, 2020</a></blockquote> `, PostTitle: "Twitter quote"},
@@ -196,8 +204,8 @@ func TestComment_Snippet(t *testing.T) {
 		{0, "", ""},
 		{-1, "test\nblah", "test blah"},
 		{5, "test\nblah", "test ..."},
-		{5, "xyz12345 xxx", "xyz12345 ..."},
+		{5, "xyz12345 xxx", "xyz12..."},
-		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 ..."},
 	}
 
 	for i, tt := range tbl {
@@ -263,7 +271,7 @@ func TestComment_sanitizeText(t *testing.T) {
 		},
 		{
 			"<a href=javascript:alert(document.domain)//>xxx</a>",
-			"&lt;a/&gt;xxx&lt;/a&gt;",
+			"xxx",
 		},
 	}
 

backend/app/store/engine/bolt.go

@@ -170,7 +170,7 @@ func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
 				return e
 			}
 
-			return bucket.ForEach(func(k, v []byte) error {
+			return bucket.ForEach(func(_, v []byte) error {
 				comment := store.Comment{}
 				if e = json.Unmarshal(v, &comment); e != nil {
 					return fmt.Errorf("failed to unmarshal: %w", e)
@@ -306,9 +306,8 @@ func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
 		})
 
 		// set read-only from age and manual bucket
-		readOnlyAge := req.ReadOnlyAge
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
-		info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+		if !info.ReadOnly && b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
-		if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
 			info.ReadOnly = true
 		}
 		return []store.PostInfo{info}, err
@@ -425,11 +424,11 @@ func (b *BoltDB) Close() error {
 }
 
 // Last returns up to max last comments for given siteID
-func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+func (b *BoltDB) lastComments(siteID string, maximum int, since time.Time) (comments []store.Comment, err error) {
 	comments = []store.Comment{}
 
-	if max > lastLimit || max == 0 {
+	if maximum > lastLimit || maximum == 0 {
-		max = lastLimit
+		maximum = lastLimit
 	}
 
 	bdb, err := b.db(siteID)
@@ -467,7 +466,7 @@ func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments
 				continue
 			}
 			comments = append(comments, comment)
-			if len(comments) >= max {
+			if len(comments) >= maximum {
 				break
 			}
 		}
@@ -725,7 +724,7 @@ func (b *BoltDB) listDetails(loc store.Locator) (result []UserDetailEntry, err e
 	err = bdb.View(func(tx *bolt.Tx) error {
 		var entry UserDetailEntry
 		bucket := tx.Bucket([]byte(userDetailsBucketName))
-		return bucket.ForEach(func(userID, value []byte) error {
+		return bucket.ForEach(func(_, value []byte) error {
 			if err = json.Unmarshal(value, &entry); err != nil {
 				return fmt.Errorf("failed to unmarshal entry: %w", e)
 			}
@@ -870,7 +869,7 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID, userID string, mode store.Dele
 		err = bdb.View(func(tx *bolt.Tx) error {
 			postsBkt := tx.Bucket([]byte(postsBucketName))
 			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+			err = postBkt.ForEach(func(_ []byte, commentVal []byte) error {
 				comment := store.Comment{}
 				if err = json.Unmarshal(commentVal, &comment); err != nil {
 					return fmt.Errorf("failed to unmarshal: %w", err)

backend/app/store/formatter.go

@@ -35,24 +35,17 @@ func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter {
 }
 
 // Format comment fields
-func (f *CommentFormatter) Format(c Comment) Comment {
+func (f *CommentFormatter) Format(c Comment, raw bool) Comment {
-	c.Text = f.FormatText(c.Text)
+	c.Text = f.FormatText(c.Text, raw)
 	return c
 }
 
 // FormatText converts text with markdown processor, applies external converters and shortens links
-func (f *CommentFormatter) FormatText(txt string) (res string) {
+//
-	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
-		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
+func (f *CommentFormatter) FormatText(txt string, raw bool) (res string) {
-		bf.BackslashLineBreak | bf.Autolink
+	mdExt, rend := GetMdExtensionsAndRenderer(raw)
-
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
-	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{
-		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
-	})
-
-	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
-
-	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(extRend)))
 	res = f.unEscape(res)
 
 	for _, conv := range f.converters {
@@ -64,14 +57,14 @@ func (f *CommentFormatter) FormatText(txt string) (res string) {
 }
 
 // Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
-func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
+func (f *CommentFormatter) shortenAutoLinks(commentHTML string, maximum int) (resHTML string) {
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("a").Each(func(i int, s *goquery.Selection) {
+	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
 		if href, ok := s.Attr("href"); ok {
-			if href != s.Text() || len(href) < max+3 || max < 3 {
+			if href != s.Text() || len(href) < maximum+3 || maximum < 3 {
 				return
 			}
 			commentURL, e := url.Parse(href)
@@ -84,7 +77,7 @@ func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTM
 				return
 			}
 
-			short := string([]rune(href)[:max-3])
+			short := string([]rune(href)[:maximum-3])
 			if len(short) < len(host) {
 				short = host
 			}
@@ -117,7 +110,7 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	if err != nil {
 		return commentHTML
 	}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	doc.Find("img").Each(func(_ int, s *goquery.Selection) {
 		s.SetAttr("loading", "lazy")
 	})
 	resHTML, err = doc.Find("body").Html()
@@ -126,3 +119,23 @@ func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
 	}
 	return resHTML
 }
+
+// GetMdExtensionsAndRenderer returns blackfriday extensions and renderer used for rendering markdown
+// within store module.
+//
+// raw=true disables SmartyPants for HTML rendering (replacement of quotes, dashes, fractions, etc).
+func GetMdExtensionsAndRenderer(raw bool) (bf.Extensions, *bfchroma.Renderer) {
+	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
+		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
+		bf.BackslashLineBreak | bf.Autolink
+
+	flags := bf.HTMLFlags(0)
+	if !raw {
+		flags = bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes
+	}
+
+	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{Flags: flags})
+
+	extRend := bfchroma.NewRenderer(bfchroma.Extend(rend), bfchroma.ChromaOptions(html.WithClasses(true)))
+	return mdExt, extRend
+}

backend/app/store/formatter_test.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -35,30 +36,38 @@ func TestFormatter_FormatText(t *testing.T) {
 			"lazy image",
 		},
 		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
+		{`no_smartpants "quoted" text`, "<p>no_smartpants &#34;quoted&#34; text</p>\n!converted", "normal quotes without smartpants"},
+		{`"quoted" text`, "<p>«quoted» text</p>\n!converted", "normal quotes with smartpants"},
+		{`no_smartpants “quoted” text`, "<p>no_smartpants “quoted” text</p>\n!converted", "curly quotes without smartpants"},
+		{`“quoted” text`, "<p>“quoted” text</p>\n!converted", "curly quotes with smartpants"},
+		{`no_smartpants «quoted» text`, "<p>no_smartpants «quoted» text</p>\n!converted", "French guillemets without smartpants"},
+		{`«quoted» text`, "<p>«quoted» text</p>\n!converted", "French guillemets with smartpants"},
 		{"smth\n```go\nfunc main(aa string) int {return 0}\n```", `<p>smth</p>
-<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
+<pre class="chroma"><code><span class="line"><span class="cl"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
 </span></span></code></pre>!converted`, "code with language"},
-		{"```\ntest_code\n```", `<pre tabindex="0" class="chroma"><code><span class="line"><span class="cl">test_code
+		{"```\ntest_code\n```", `<pre class="chroma"><code><span class="line"><span class="cl">test_code
 </span></span></code></pre>!converted`, "code without language"},
 	}
 	f := NewCommentFormatter(mockConverter{})
 	for _, tt := range tbl {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.out, f.FormatText(tt.in))
+			raw := strings.HasPrefix(tt.in, `no_smartpants`)
+			t.Logf("raw: %v", raw)
+			assert.Equal(t, tt.out, f.FormatText(tt.in, raw))
 		})
 	}
 }
 
 func TestFormatter_FormatTextNoConverter(t *testing.T) {
 	f := NewCommentFormatter()
-	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatTextConverterFunc(t *testing.T) {
 	fn := CommentConverterFunc(func(text string) string { return "zz!" + text })
 	f := NewCommentFormatter(fn)
-	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345"))
+	assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345", false))
 }
 
 func TestFormatter_FormatComment(t *testing.T) {
@@ -78,7 +87,7 @@ func TestFormatter_FormatComment(t *testing.T) {
 	f := NewCommentFormatter(mockConverter{})
 	exp := comment
 	exp.Text = "<p>blah</p>\n\n<p>xyz</p>\n!converted"
-	assert.Equal(t, exp, f.Format(comment))
+	assert.Equal(t, exp, f.Format(comment, false))
 }
 
 func TestFormatter_ShortenAutoLinks(t *testing.T) {

backend/app/store/image/bolt_store.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"time"
 
@@ -118,6 +119,18 @@ func (b *Bolt) Load(id string) ([]byte, error) {
 	return data, nil
 }
 
+// Delete image from storage
+func (b *Bolt) Delete(id string) error {
+	return b.db.Update(func(tx *bolt.Tx) error {
+		// deleting a non-existing key doesn't return an error, so joining errors from deleting an image
+		// from both buckets is safe and will return nil if there are no errors on the real delete
+		// or image is absent in both buckets
+		err := tx.Bucket([]byte(imagesBktName)).Delete([]byte(id))
+		err = errors.Join(err, tx.Bucket([]byte(imagesStagedBktName)).Delete([]byte(id)))
+		return err
+	})
+}
+
 // Cleanup runs scan of staging and removes old data based on ttl
 func (b *Bolt) Cleanup(_ context.Context, ttl time.Duration) error {
 	return b.db.Update(func(tx *bolt.Tx) error {

backend/app/store/image/bolt_store_test.go

@@ -58,6 +58,36 @@ func TestBoltStore_LoadAfterSave(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestBoltStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareBoltImageStorageTest(t)
+	defer teardown()
+
+	// delete image from permanent storage
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// delete staging image
+	id = "staging_img"
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	err = svc.Delete(id)
+	assert.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestBoltStore_Cleanup(t *testing.T) {
 	svc, teardown := prepareBoltImageStorageTest(t)
 	defer teardown()

backend/app/store/image/fs_store.go

@@ -23,7 +23,8 @@ type FileSystem struct {
 	Staging    string
 	Partitions int
 
-	crc struct {
+	moveLock sync.Mutex // needed only for deleting images or moving them from staging to permanent storage
+	crc      struct {
 		*crc64.Table
 		sync.Once
 		mask    string
@@ -50,6 +51,8 @@ func (f *FileSystem) Save(id string, img []byte) error {
 
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 	log.Printf("[DEBUG] Commit image %s", id)
 	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
 
@@ -107,11 +110,31 @@ func (f *FileSystem) Load(id string) ([]byte, error) {
 	return io.ReadAll(fh)
 }
 
+// Delete image from storage
+func (f *FileSystem) Delete(id string) error {
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
+	staging := f.location(f.Staging, id)
+	// file doesn't exist on staging, delete from permanent location
+	if _, err := os.Stat(staging); os.IsNotExist(err) {
+		file := f.location(f.Location, id)
+		e := os.Remove(file)
+		_ = os.Remove(path.Dir(file)) // try to remove directory
+		return e
+	}
+	// delete file from staging
+	err := os.Remove(staging)
+	_ = os.Remove(path.Dir(staging)) // try to remove directory
+	return err
+}
+
 // Cleanup runs scan of staging and removes old files based on ttl
 func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error {
 	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
 		return nil
 	}
+	f.moveLock.Lock()
+	defer f.moveLock.Unlock()
 
 	// we can ignore context as on local FS remove is relatively fast operation
 	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
@@ -143,7 +166,7 @@ func (f *FileSystem) Info() (StoreInfo, error) {
 	}
 
 	var ts time.Time
-	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+	err := filepath.Walk(f.Staging, func(_ string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}

backend/app/store/image/fs_store_test.go

@@ -131,6 +131,31 @@ func TestFsStore_LoadAfterCommit(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestFsStore_LoadAfterDelete(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id := "test_img"
+	err := svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+
+	// create file on staging
+	err = svc.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+	err = svc.Delete(id)
+	require.NoError(t, err)
+
+	_, err = svc.Load(id)
+	assert.Error(t, err)
+}
+
 func TestFsStore_location(t *testing.T) {
 	tbl := []struct {
 		partitions int

backend/app/store/image/image.go

@@ -70,6 +70,7 @@ type Store interface {
 	Info() (StoreInfo, error)         // get meta information about storage
 	Save(id string, img []byte) error // store image with passed id to staging
 	Load(id string) ([]byte, error)   // load image by ID
+	Delete(id string) error           // delete image by ID
 
 	ResetCleanupTimer(id string) error                    // resets cleanup timer for the image, called on comment preview
 	Commit(id string) error                               // move image from staging to permanent
@@ -88,8 +89,8 @@ func NewService(s Store, p ServiceParams) *Service {
 	return &Service{ServiceParams: p, store: s}
 }
 
-// SubmitAndCommit multiple ids immediately
+// Commit multiple ids immediately
-func (s *Service) SubmitAndCommit(idsFn func() []string) error {
+func (s *Service) Commit(idsFn func() []string) error {
 	errs := new(multierror.Error)
 	for _, id := range idsFn() {
 		err := s.store.Commit(id)
@@ -117,7 +118,7 @@ func (s *Service) Submit(idsFn func() []string) {
 				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.EditDuration {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
-				err := s.SubmitAndCommit(req.idsFn)
+				err := s.Commit(req.idsFn)
 				if err != nil {
 					log.Printf("[WARN] image commit error %v", err)
 				}
@@ -141,39 +142,14 @@ func (s *Service) Submit(idsFn func() []string) {
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
 func (s *Service) ExtractPictures(commentHTML string) (ids []string) {
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	return s.extractImageIDs(commentHTML, true)
-	if err != nil {
+}
-		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
-		return nil
-	}
-	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
-		if im, ok := sl.Attr("src"); ok {
-			if strings.Contains(im, s.ImageAPI) {
-				elems := strings.Split(im, "/")
-				if len(elems) >= 2 {
-					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
-					ids = append(ids, id)
-				}
-			}
-			if strings.Contains(im, s.ProxyAPI) {
-				proxiedURL, err := url.Parse(im)
-				if err != nil {
-					return
-				}
-				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
-				if err != nil {
-					return
-				}
-				imgID, err := CachedImgID(string(imgURL))
-				if err != nil {
-					return
-				}
-				ids = append(ids, imgID)
-			}
-		}
-	})
 
-	return ids
+// ExtractNonProxiedPictures gets list of non-proxied images from the doc html and convert from urls to ids, i.e. user/pic.png
+// This method is used in image check on post preview and load, as proxied images have lazy loading
+// and wouldn't be present on disk but still valid as they will be loaded the first time someone requests them.
+func (s *Service) ExtractNonProxiedPictures(commentHTML string) (ids []string) {
+	return s.extractImageIDs(commentHTML, false)
 }
 
 // Cleanup runs periodic cleanup with 1.5*ServiceParams.EditDuration. Blocking loop, should be called inside of goroutine by consumer
@@ -237,6 +213,11 @@ func (s *Service) Load(id string) ([]byte, error) {
 	return s.store.Load(id)
 }
 
+// Delete wraps storage Delete function.
+func (s *Service) Delete(id string) error {
+	return s.store.Delete(id)
+}
+
 // Save wraps storage Save function, validating and resizing the image before calling it.
 func (s *Service) Save(userID string, r io.Reader) (id string, err error) {
 	id = path.Join(userID, guid())
@@ -262,6 +243,43 @@ func (s *Service) ImgContentType(img []byte) string {
 	return contentType
 }
 
+// returns list of image IDs from the comment html, including proxied images if includeProxied is true
+func (s *Service) extractImageIDs(commentHTML string, includeProxied bool) (ids []string) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err)
+		return nil
+	}
+	doc.Find("img").Each(func(_ int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					ids = append(ids, id)
+				}
+			}
+			if includeProxied && strings.Contains(im, s.ProxyAPI) {
+				proxiedURL, err := url.Parse(im)
+				if err != nil {
+					return
+				}
+				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
+				if err != nil {
+					return
+				}
+				imgID, err := CachedImgID(string(imgURL))
+				if err != nil {
+					return
+				}
+				ids = append(ids, imgID)
+			}
+		}
+	})
+
+	return ids
+}
+
 // prepareImage calls readAndValidateImage and resize on provided image.
 func (s *Service) prepareImage(r io.Reader) ([]byte, error) {
 	data, err := readAndValidateImage(r, s.MaxSize)

backend/app/store/image/image_mock.go

@@ -4,9 +4,9 @@
 package image
 
 import (
-	context "context"
+	"context"
 	"sync"
-	time "time"
+	"time"
 )
 
 // Ensure, that StoreMock does implement Store.
@@ -25,6 +25,9 @@ var _ Store = &StoreMock{}
 //			CommitFunc: func(id string) error {
 //				panic("mock out the Commit method")
 //			},
+//			DeleteFunc: func(id string) error {
+//				panic("mock out the Delete method")
+//			},
 //			InfoFunc: func() (StoreInfo, error) {
 //				panic("mock out the Info method")
 //			},
@@ -50,6 +53,9 @@ type StoreMock struct {
 	// CommitFunc mocks the Commit method.
 	CommitFunc func(id string) error
 
+	// DeleteFunc mocks the Delete method.
+	DeleteFunc func(id string) error
+
 	// InfoFunc mocks the Info method.
 	InfoFunc func() (StoreInfo, error)
 
@@ -76,6 +82,11 @@ type StoreMock struct {
 			// ID is the id argument value.
 			ID string
 		}
+		// Delete holds details about calls to the Delete method.
+		Delete []struct {
+			// ID is the id argument value.
+			ID string
+		}
 		// Info holds details about calls to the Info method.
 		Info []struct {
 		}
@@ -99,6 +110,7 @@ type StoreMock struct {
 	}
 	lockCleanup           sync.RWMutex
 	lockCommit            sync.RWMutex
+	lockDelete            sync.RWMutex
 	lockInfo              sync.RWMutex
 	lockLoad              sync.RWMutex
 	lockResetCleanupTimer sync.RWMutex
@@ -173,6 +185,38 @@ func (mock *StoreMock) CommitCalls() []struct {
 	return calls
 }
 
+// Delete calls DeleteFunc.
+func (mock *StoreMock) Delete(id string) error {
+	if mock.DeleteFunc == nil {
+		panic("StoreMock.DeleteFunc: method is nil but Store.Delete was just called")
+	}
+	callInfo := struct {
+		ID string
+	}{
+		ID: id,
+	}
+	mock.lockDelete.Lock()
+	mock.calls.Delete = append(mock.calls.Delete, callInfo)
+	mock.lockDelete.Unlock()
+	return mock.DeleteFunc(id)
+}
+
+// DeleteCalls gets all the calls that were made to Delete.
+// Check the length with:
+//
+//	len(mockedStore.DeleteCalls())
+func (mock *StoreMock) DeleteCalls() []struct {
+	ID string
+} {
+	var calls []struct {
+		ID string
+	}
+	mock.lockDelete.RLock()
+	calls = mock.calls.Delete
+	mock.lockDelete.RUnlock()
+	return calls
+}
+
 // Info calls InfoFunc.
 func (mock *StoreMock) Info() (StoreInfo, error) {
 	if mock.InfoFunc == nil {

backend/app/store/image/image_test.go

@@ -19,10 +19,10 @@ import (
 
 func TestService_SaveAndLoad(t *testing.T) {
 	store := StoreMock{
-		SaveFunc: func(id string, img []byte) error {
+		SaveFunc: func(string, []byte) error {
 			return nil
 		},
-		LoadFunc: func(id string) ([]byte, error) {
+		LoadFunc: func(string) ([]byte, error) {
 			return nil, nil
 		},
 	}
@@ -105,6 +105,7 @@ func TestService_ExtractPictures(t *testing.T) {
 	ids = svc.ExtractPictures(html)
 	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
+	require.Empty(t, svc.ExtractNonProxiedPictures(html), "no non-proxied images expected to be found")
 
 	// bad url
 	html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
@@ -125,7 +126,7 @@ func TestService_ExtractPictures(t *testing.T) {
 
 func TestService_Cleanup(t *testing.T) {
 	store := StoreMock{
-		CleanupFunc: func(ctx context.Context, ttl time.Duration) error {
+		CleanupFunc: func(context.Context, time.Duration) error {
 			return nil
 		},
 	}
@@ -140,17 +141,13 @@ func TestService_Cleanup(t *testing.T) {
 
 func TestService_Submit(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
+		CommitFunc:            func(string) error { return nil },
-			return nil
+		ResetCleanupTimerFunc: func(string) error { return nil },
-		},
-		ResetCleanupTimerFunc: func(id string) error {
-			return nil
-		},
 	}
 	svc := NewService(&store, ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100})
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	assert.Equal(t, 3, len(store.ResetCleanupTimerCalls()))
-	err := svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} })
+	err := svc.Commit(func() []string { return []string{"id4", "id5"} })
 	assert.NoError(t, err)
 	svc.Submit(func() []string { return []string{"id6", "id7"} })
 	assert.Equal(t, 5, len(store.ResetCleanupTimerCalls()))
@@ -163,12 +160,8 @@ func TestService_Submit(t *testing.T) {
 
 func TestService_Close(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
+		CommitFunc:            func(string) error { return nil },
-			return nil
+		ResetCleanupTimerFunc: func(string) error { return nil },
-		},
-		ResetCleanupTimerFunc: func(id string) error {
-			return nil
-		},
 	}
 	svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Hour * 24}}
 	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
@@ -181,10 +174,8 @@ func TestService_Close(t *testing.T) {
 
 func TestService_SubmitDelay(t *testing.T) {
 	store := StoreMock{
-		CommitFunc: func(id string) error {
+		CommitFunc: func(string) error { return nil },
-			return nil
+		ResetCleanupTimerFunc: func(string) error {
-		},
-		ResetCleanupTimerFunc: func(id string) error {
 			return nil
 		},
 	}
@@ -287,7 +278,7 @@ func TestCachedImgID(t *testing.T) {
 	assert.Equal(t, "cached_images/"+Sha1Str("example.org")+"-"+Sha1Str(imgURL), img)
 }
 
-func TestService_DoubleClose(t *testing.T) {
+func TestService_DoubleClose(*testing.T) {
 	store := StoreMock{}
 	svc := NewService(&store, ServiceParams{EditDuration: 20 * time.Millisecond})
 	svc.Close(context.TODO())

backend/app/store/image/remote_store.go

@@ -41,6 +41,12 @@ func (r *RPC) Load(id string) ([]byte, error) {
 	return io.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg)))
 }
 
+// Delete image from storage
+func (r *RPC) Delete(id string) error {
+	_, err := r.Call("image.delete", id)
+	return err
+}
+
 // Commit file stored in staging location by moving it to permanent location
 func (r *RPC) Commit(id string) error {
 	_, err := r.Call("image.commit", id)

backend/app/store/image/remote_store_test.go

@@ -41,6 +41,18 @@ func TestRemote_Load(t *testing.T) {
 	assert.Equal(t, gopherPNGBytes(), res)
 }
 
+func TestRemote_Delete(t *testing.T) {
+	ts := testServer(t, `{"method":"image.delete","params":"54321","id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	err := c.Delete("54321")
+	assert.NoError(t, err)
+}
+
 func TestRemote_Commit(t *testing.T) {
 	ts := testServer(t, `{"method":"image.commit","params":"gopher_id","id":1}`, `{"id":1}`)
 	defer ts.Close()

backend/app/store/service/service.go

@@ -5,15 +5,17 @@ package service
 import (
 	"fmt"
 	"math"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-multierror"
+	bf "github.com/russross/blackfriday/v2"
 
 	"github.com/umputun/remark42/backend/app/store"
 	"github.com/umputun/remark42/backend/app/store/admin"
@@ -26,6 +28,7 @@ type DataStore struct {
 	Engine              engine.Interface
 	EditDuration        time.Duration
 	AdminStore          admin.Store
+	MinCommentSize      int
 	MaxCommentSize      int
 	MaxVotes            int
 	RestrictSameIPVotes struct {
@@ -46,7 +49,7 @@ type DataStore struct {
 	}
 
 	repliesCache struct {
-		lcw.LoadingCache
+		lcw.LoadingCache[struct{}]
 		once sync.Once
 	}
 }
@@ -68,7 +71,7 @@ type PostMetaData struct {
 	ReadOnly bool   `json:"read_only"`
 }
 
-const defaultCommentMaxSize = 2000
+const defaultCommentMaxSize = 2048
 const maxLastCommentsReply = 5000
 
 // UnlimitedVotes doesn't restrict MaxVotes
@@ -278,7 +281,7 @@ func (s *DataStore) submitImages(comment store.Comment) {
 
 	var err error
 	if comment.Imported {
-		err = s.ImageService.SubmitAndCommit(idsFn)
+		err = s.ImageService.Commit(idsFn)
 	} else {
 		s.ImageService.Submit(idsFn)
 	}
@@ -531,7 +534,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 func (s *DataStore) HasReplies(comment store.Comment) bool {
 	s.repliesCache.once.Do(func() {
 		// default expiration time of 5 minutes and cleanup time of 2.5 minutes
-		s.repliesCache.LoadingCache, _ = lcw.NewExpirableCache(lcw.TTL(5 * time.Minute))
+		o := lcw.NewOpts[struct{}]()
+		s.repliesCache.LoadingCache, _ = lcw.NewExpirableCache[struct{}](o.TTL(5 * time.Minute))
 	})
 
 	if _, found := s.repliesCache.Peek(comment.ID); found {
@@ -551,7 +555,7 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 				// When this code is reached, key "comment.ID" is not in cache.
 				// Calling cache.Get on it will put it in cache with 5 minutes TTL.
 				// We call it with empty struct as value as we care about keys and not values.
-				_, _ = s.repliesCache.Get(comment.ID, func() (interface{}, error) { return struct{}{}, nil })
+				_, _ = s.repliesCache.Get(comment.ID, func() (struct{}, error) { return struct{}{}, nil })
 				return true
 			}
 		}
@@ -627,7 +631,9 @@ func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, e
 	return res, nil
 }
 
-// ValidateComment checks if comment size below max and user fields set
+// ValidateComment checks if comment size below max and user fields set.
+// It also validates the absence of relative links as they are almost never the intention of the commenter,
+// usually added by mistakes and only create confusion.
 func (s *DataStore) ValidateComment(c *store.Comment) error {
 	maxSize := s.MaxCommentSize
 	if s.MaxCommentSize <= 0 {
@@ -636,13 +642,33 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
 	if c.Orig == "" {
 		return fmt.Errorf("empty comment text")
 	}
-	if len([]rune(c.Orig)) > maxSize {
+	commentLength := len([]rune(c.Orig))
-		return fmt.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, len([]rune(c.Orig)))
+	if commentLength > maxSize {
+		return fmt.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, commentLength)
+	}
+	if s.MinCommentSize > 0 && commentLength < s.MinCommentSize {
+		return fmt.Errorf("comment text is smaller than min allowed size %d (%d)", s.MinCommentSize, commentLength)
 	}
 	if c.User.ID == "" || c.User.Name == "" {
 		return fmt.Errorf("empty user info")
 	}
-	return nil
+
+	// for validation purposes it's not important if SmartyPants formatting is disabled or enabled,
+	// while for storing the comment that flag is set based on user preference
+	mdExt, rend := store.GetMdExtensionsAndRenderer(false)
+	parser := bf.New(bf.WithRenderer(rend), bf.WithExtensions(bf.CommonExtensions), bf.WithExtensions(mdExt))
+	var wrongLinkError error
+	parser.Parse([]byte(c.Orig)).Walk(func(node *bf.Node, _ bool) bf.WalkStatus {
+		if len(node.LinkData.Destination) != 0 &&
+			!(strings.HasPrefix(string(node.LinkData.Destination), "http://") ||
+				strings.HasPrefix(string(node.LinkData.Destination), "https://") ||
+				strings.HasPrefix(string(node.LinkData.Destination), "mailto:")) {
+			wrongLinkError = fmt.Errorf("links should start with mailto:, http:// or https://")
+			return bf.Terminate
+		}
+		return bf.GoToNext
+	})
+	return wrongLinkError
 }
 
 // IsAdmin checks if usesID in the list of admins
@@ -737,7 +763,23 @@ func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo
 	if len(res) == 0 {
 		return store.PostInfo{}, fmt.Errorf("post %+v not found", locator)
 	}
-	return res[0], nil
+	// URL request
+	if locator.URL != "" {
+		return res[0], nil
+	}
+	// site-wide request which returned multiple store.PostInfo, so that URL and ReadOnly flags don't make sense
+	var info store.PostInfo
+	for _, i := range res {
+		info.Count += i.Count
+		if info.FirstTS.IsZero() || i.FirstTS.Before(info.FirstTS) {
+			info.FirstTS = i.FirstTS
+		}
+		if info.LastTS.IsZero() || i.LastTS.After(info.LastTS) {
+			info.LastTS = i.LastTS
+		}
+	}
+	return info, nil
+
 }
 
 // Delete comment by id
@@ -756,6 +798,34 @@ func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.D
 		s.repliesCache.Delete(commentID)
 		s.repliesCache.Delete(comment.ParentID)
 	}
+
+	// delete images from the comment if they are not reused elsewhere in comments to the same page
+	idsFn := func() []string { // get IDs of all images from the same URL to verify if image from deleted comment was reused
+		comments, e := s.Engine.Find(engine.FindRequest{Locator: locator})
+		if e != nil {
+			log.Printf("[WARN] can't get comments %s text for deleted comment image check, %v", comment.ID, err)
+			return nil
+		}
+		var imgIDs = []string{}
+		for _, cc := range comments {
+			// exclude the comment we are deleting
+			if cc.ID != commentID {
+				imgIDs = append(imgIDs, s.ImageService.ExtractPictures(cc.Text)...)
+			}
+		}
+		return imgIDs
+	}
+	commentImgIDs := s.ImageService.ExtractPictures(comment.Text)
+	pageImgIDs := idsFn()
+	for _, id := range commentImgIDs {
+		if !slices.Contains(pageImgIDs, id) {
+			if err := s.ImageService.Delete(id); err != nil {
+				log.Printf("[WARN] failed to delete image %s on comment %s deletion, %v", id, commentID, err)
+			}
+		}
+	}
+	log.Printf("[ERROR] commentImgIDs: %v, pageImgIDs: %v", commentImgIDs, pageImgIDs)
+
 	req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
 	return s.Engine.Delete(req)
 }

backend/app/store/service/service_test.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -131,7 +132,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
 	b := DataStore{Engine: eng, AdminStore: ks,
-		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})}
 	defer b.Close()
 
 	postPath := "/post/42"
@@ -171,6 +172,50 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
 }
 
+func TestService_Put(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, AdminStore: ks}
+
+	comment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "test text",
+		User:      store.User{ID: "user2", Name: "user name 2"},
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+	}
+	_, err := b.Create(comment)
+	require.NoError(t, err)
+
+	// create new comment with everything different to replace the first one with fields below
+	updatedComment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-new",
+		Text:      "new text",
+		User:      store.User{ID: "user3", Name: "user name 3"},
+		Locator:   store.Locator{URL: "https://example.com", SiteID: "example"},
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+	}
+
+	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, updatedComment)
+	require.NoError(t, err)
+
+	// request with wrong user, should not affect the comment user
+	got, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", store.User{ID: "user1", Name: "user name 1"})
+	require.NoError(t, err)
+	assert.Equal(t, "c-1", got.ID)
+	assert.Equal(t, "new text", got.Text)
+	assert.Equal(t, "id-1", got.ParentID, "should be unaltered")
+	assert.Equal(t, "user2", got.User.ID, "should be unaltered")
+	assert.Equal(t, "user name 2", got.User.Name, "should be unaltered")
+	assert.Equal(t, "https://radio-t.com", got.Locator.URL, "should be unaltered")
+	assert.Equal(t, "radio-t", got.Locator.SiteID, "should be unaltered")
+	assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), got.Timestamp, "should be unaltered")
+
+}
+
 func TestService_SetTitle(t *testing.T) {
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -195,7 +240,7 @@ func TestService_SetTitle(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
 	b := DataStore{Engine: eng, AdminStore: ks,
-		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})}
 	defer b.Close()
 	comment := store.Comment{
 		Text:      "text",
@@ -413,7 +458,6 @@ func TestService_VoteAggressive(t *testing.T) {
 	assert.Equal(t, 0, len(res[0].VotedIPs), "vote ips hidden")
 
 	// random +1/-1 result should be [0..2]
-	rand.Seed(time.Now().UnixNano())
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
 		go func() {
@@ -546,6 +590,111 @@ func TestService_VoteControversy(t *testing.T) {
 	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
 }
 
+func TestService_RestrictedWords(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	restictedWordLister := StaticRestrictedWordsLister{Words: []string{"restricted"}}
+	b := DataStore{Engine: eng, AdminStore: ks, RestrictedWordsMatcher: NewRestrictedWordsMatcher(restictedWordLister)}
+
+	// test creating a comment with restricted words which should fail with appropriate error
+	reply := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "restricted word",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+	id, err := b.Create(reply)
+	assert.EqualError(t, err, ErrRestrictedWordsFound.Error(), "should fail with RestrictedWordError")
+	assert.Empty(t, id)
+}
+
+func TestDataStore_AdminStoreErrors(t *testing.T) {
+	badKey := true
+	badEnabled := true
+	as := admin.StoreMock{
+		OnEventFunc: func(string, admin.EventType) error { return errors.New("err") },
+		KeyFunc: func(string) (string, error) {
+			if badKey {
+				return "", errors.New("mock key err")
+			}
+			return "secret", nil
+		},
+		EnabledFunc: func(string) (bool, error) {
+			if badEnabled {
+				return false, errors.New("mock enabled err")
+			}
+			return true, nil
+		},
+		AdminsFunc: func(string) ([]string, error) { return nil, errors.New("mock admins err") },
+	}
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, AdminStore: &as, MaxVotes: -1}
+	comment := store.Comment{
+		ID:        "c-1",
+		ParentID:  "id-1",
+		Text:      "restricted word",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+
+	// Key call error
+	id, err := b.Create(comment)
+	assert.ErrorContainsf(t, err, "mock key err", "should fail with mock error")
+	assert.Empty(t, id)
+	assert.Equal(t, len(as.KeyCalls()), 1)
+	assert.Equal(t, len(as.EnabledCalls()), 0)
+	assert.Equal(t, len(as.OnEventCalls()), 0)
+
+	// Enabled call error
+	badKey = false
+	id, err = b.Create(comment)
+	assert.ErrorContains(t, err, "mock enabled err", "should fail with mock error")
+	assert.Empty(t, id)
+	assert.Equal(t, len(as.KeyCalls()), 2)
+	assert.Equal(t, len(as.EnabledCalls()), 1)
+	assert.Equal(t, len(as.OnEventCalls()), 0)
+
+	// only OnEvent error
+	badEnabled = false
+	id, err = b.Create(comment)
+	assert.NoError(t, err, "OnEvent error should be just logged")
+	assert.Equal(t, id, "c-1")
+	assert.Equal(t, len(as.KeyCalls()), 3)
+	assert.Equal(t, len(as.EnabledCalls()), 2)
+	assert.Equal(t, len(as.OnEventCalls()), 1)
+
+	// OnEvent error on Vote call
+	_, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "c-1",
+		UserID: "user4", Val: true})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+	assert.Equal(t, len(as.KeyCalls()), 4)
+	assert.Equal(t, len(as.EnabledCalls()), 3)
+	assert.Equal(t, len(as.OnEventCalls()), 2)
+
+	// Admins error
+	isAdmin := b.IsAdmin("radio-t", "user2")
+	assert.False(t, isAdmin)
+	assert.Equal(t, len(as.AdminsCalls()), 1)
+	assert.Equal(t, len(as.OnEventCalls()), 2)
+
+	// OnEvent error on EditComment call
+	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", EditRequest{Text: "new text"})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+
+	// OnEvent error on Delete call
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", store.SoftDelete)
+	assert.NoError(t, err, "OnEvent error should be just logged")
+
+	// OnEvent error on EditComment Delete call
+	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "c-1", EditRequest{Delete: true})
+	assert.NoError(t, err, "OnEvent error should be just logged")
+}
+
 func TestService_VoteSameIP(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
@@ -775,27 +924,31 @@ func TestService_EditCommentAdmin(t *testing.T) {
 }
 
 func TestService_ValidateComment(t *testing.T) {
-	b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{MinCommentSize: 6, MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
 	longText := fmt.Sprintf("%4000s", "X")
 
 	tbl := []struct {
 		inp store.Comment
-		err error
+		err string
 	}{
-		{inp: store.Comment{}, err: fmt.Errorf("empty comment text")},
+		{inp: store.Comment{}, err: "empty comment text"},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: nil},
+		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: ""},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: fmt.Errorf("empty user info")},
+		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: "empty user info"},
-		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: fmt.Errorf("comment text exceeded max allowed size 2000 (4000)")},
+		{inp: store.Comment{Orig: "short", User: store.User{ID: "myid", Name: "name"}}, err: "comment text is smaller than min allowed size 6 (5)"},
+		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: "comment text exceeded max allowed size 2000 (4000)"},
+		{inp: store.Comment{Orig: "here is a link with relative URL: [google.com](url)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
+		{inp: store.Comment{Orig: "here is a link with relative URL: [google.com](url)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
+		{inp: store.Comment{Orig: "multiple links, one is bad: [test](http://test) [test2](bad_url) [test3](https://test3)", User: store.User{ID: "myid", Name: "name"}}, err: "links should start with mailto:, http:// or https://"},
 	}
 
 	for n, tt := range tbl {
 		err := b.ValidateComment(&tt.inp)
-		if tt.err == nil {
+		if tt.err == "" {
 			assert.NoError(t, err, "check #%d", n)
 			continue
 		}
 		require.Error(t, err)
-		assert.EqualError(t, tt.err, err.Error(), "check #%d", n)
+		assert.EqualError(t, err, tt.err, "check #%d", n)
 	}
 }
 
@@ -1154,7 +1307,7 @@ func TestService_Find(t *testing.T) {
 	assert.InDelta(t, 0, res[1].Controversy, 0.01)
 
 	// make sure title sanitized
-	assert.Equal(t, "some title, &lt;a href=\"http://radio-t.com\" rel=\"nofollow\"&gt;link&lt;/a&gt;", res[0].PostTitle)
+	assert.Equal(t, "some title, link", res[0].PostTitle)
 }
 
 func TestService_FindSince(t *testing.T) {
@@ -1183,18 +1336,55 @@ func TestService_Info(t *testing.T) {
 	b := DataStore{Engine: eng, EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", nil, []string{"user2"}, "user@email.com")}
 
-	info, err := b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
+	// add one more comment for another URL to test site-wide Info request
+	comment := store.Comment{
+		ID:        "123456xyz",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/another", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	require.NoError(t, err)
+
+	// get non-existing URL info
+	info, err := b.Info(store.Locator{URL: "bad", SiteID: "radio-t"}, 0)
+	assert.Error(t, err)
+	assert.Empty(t, info)
+
+	// get non-existing site info
+	info, err = b.Info(store.Locator{SiteID: "bad"}, 0)
+	assert.Error(t, err)
+	assert.Empty(t, info)
+
+	// test two initially created comments and store first comment FirstTS
+	info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
 	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com", info.URL)
 	assert.Equal(t, 2, info.Count)
 	assert.False(t, info.ReadOnly)
 	assert.True(t, info.LastTS.After(info.FirstTS))
+	firstTS := info.FirstTS
 
 	time.Sleep(1 * time.Second) // make post RO in 1sec
 	info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 1)
 	require.NoError(t, err)
 	assert.Equal(t, "https://radio-t.com", info.URL)
 	assert.True(t, info.ReadOnly)
+
+	// get last created comment LastTS
+	info, err = b.Info(store.Locator{URL: "https://radio-t.com/another", SiteID: "radio-t"}, 0)
+	require.NoError(t, err)
+	lastTS := info.LastTS
+
+	// site-level request
+	info, err = b.Info(store.Locator{SiteID: "radio-t"}, 1)
+	require.NoError(t, err)
+	assert.Equal(t, 3, info.Count)
+	assert.Empty(t, info.URL, "site-level request should not set URL")
+	assert.False(t, info.ReadOnly, "site-level request should not set ReadOnly")
+	assert.Equal(t, firstTS, info.FirstTS, "site-level request should have FirstTS from the first post")
+	assert.Equal(t, lastTS, info.LastTS, "site-level request should have LastTS from the last post")
 }
 
 func TestService_Delete(t *testing.T) {
@@ -1216,6 +1406,80 @@ func TestService_Delete(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestService_deleteImagesOnCommentDelete(t *testing.T) {
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	mockStore := image.StoreMock{
+		DeleteFunc:            func(string) error { return nil },
+		CommitFunc:            func(string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
+	}
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 50 * time.Millisecond,
+			ImageAPI:     "/images/dev/",
+			ProxyAPI:     "/non_existent",
+		})
+	defer imgSvc.Close(context.TODO())
+
+	// two comments for https://radio-t.com
+	eng, teardown := prepStoreEngine(t)
+	defer teardown()
+	b := DataStore{Engine: eng, EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+	b.submitImages(c)
+	// reply to the first comment with one new image and one existing one
+	c = store.Comment{
+		ID:       "id-23",
+		ParentID: "id-22",
+		Text:     `some text <img src="/images/dev/pic2.png"/> xx <img src="/images/dev/pic3.png"/>`,
+		Locator:  store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:     store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Engine.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+	b.submitImages(c)
+
+	// verify that images are in staging store
+	assert.Equal(t, 4, len(mockStore.ResetCleanupTimerCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.ResetCleanupTimerCalls()[0].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.ResetCleanupTimerCalls()[1].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.ResetCleanupTimerCalls()[2].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.ResetCleanupTimerCalls()[3].ID)
+	time.Sleep(b.EditDuration + 100*time.Millisecond)
+	// verify that they got into the main store
+	assert.Equal(t, 4, len(mockStore.CommitCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.CommitCalls()[0].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.CommitCalls()[1].ID)
+	assert.Equal(t, "dev/pic2.png", mockStore.CommitCalls()[2].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.CommitCalls()[3].ID)
+
+	// delete the first comment
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-22", store.SoftDelete)
+	assert.NoError(t, err)
+	// verify that images are deleted from the main store
+	assert.Equal(t, 1, len(mockStore.DeleteCalls()))
+	assert.Equal(t, "dev/pic1.png", mockStore.DeleteCalls()[0].ID)
+
+	// delete the second comment
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-23", store.SoftDelete)
+	assert.NoError(t, err)
+	// verify that images are deleted from the main store
+	assert.Equal(t, 3, len(mockStore.DeleteCalls()))
+	assert.Equal(t, "dev/pic2.png", mockStore.DeleteCalls()[1].ID)
+	assert.Equal(t, "dev/pic3.png", mockStore.DeleteCalls()[2].ID)
+}
+
 // DeleteUser removes all comments from user
 func TestService_DeleteUser(t *testing.T) {
 	// two comments for https://radio-t.com, no reply
@@ -1396,8 +1660,8 @@ func TestService_submitImages(t *testing.T) {
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
 	mockStore := image.StoreMock{
-		CommitFunc:            func(id string) error { return nil },
+		CommitFunc:            func(string) error { return nil },
-		ResetCleanupTimerFunc: func(id string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	imgSvc := image.NewService(&mockStore,
 		image.ServiceParams{
@@ -1438,10 +1702,10 @@ func TestService_ResubmitStagingImages(t *testing.T) {
 		InfoFunc: func() (image.StoreInfo, error) {
 			return image.StoreInfo{FirstStagingImageTS: time.Time{}.Add(time.Second)}, nil
 		},
-		CommitFunc: func(id string) error {
+		CommitFunc: func(string) error {
 			return nil
 		},
-		ResetCleanupTimerFunc: func(id string) error { return nil },
+		ResetCleanupTimerFunc: func(string) error { return nil },
 	}
 	imgSvc := image.NewService(&mockStore,
 		image.ServiceParams{
@@ -1533,7 +1797,7 @@ func TestService_ResubmitStagingImages_EngineError(t *testing.T) {
 
 	first := true
 	engineMock := engine.InterfaceMock{
-		FindFunc: func(req engine.FindRequest) ([]store.Comment, error) {
+		FindFunc: func(engine.FindRequest) ([]store.Comment, error) {
 			if first {
 				first = false
 				return nil, nil
@@ -1558,7 +1822,7 @@ func TestService_ResubmitStagingImages_EngineError(t *testing.T) {
 
 func TestService_alterComment(t *testing.T) {
 	engineMock := engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			return false, nil
 		},
 	}
@@ -1580,7 +1844,7 @@ func TestService_alterComment(t *testing.T) {
 
 	first := true
 	engineMock = engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			if first {
 				first = false
 				return false, nil
@@ -1598,7 +1862,7 @@ func TestService_alterComment(t *testing.T) {
 
 	first = true
 	engineMock = engine.InterfaceMock{
-		FlagFunc: func(req engine.FlagRequest) (bool, error) {
+		FlagFunc: func(engine.FlagRequest) (bool, error) {
 			if first {
 				first = false
 				return true, nil
@@ -1656,10 +1920,10 @@ func TestService_DoubleClose_Static(t *testing.T) {
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
 	b := DataStore{Engine: eng, AdminStore: ks,
-		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
+		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})}
-	b.Close()
+	assert.NoError(t, b.Close())
 	// second call should not result in panic or errors
-	b.Close()
+	assert.NoError(t, b.Close())
 }
 
 // makes new boltdb, put two records

backend/app/store/service/testdata/tree.json

@@ -242,11 +242,5 @@
                   "time": "2017-12-25T19:47:22Z"
               }
           }
-      ],
+      ]
-      "info": {
+  }
-          "url": "url",
-          "count": 12,
-          "first_time": "2017-12-25T19:46:01Z",
-          "last_time": "2017-12-25T19:47:22Z"
-      }
-  }

backend/app/store/service/testdata/tree_del.json

@@ -176,11 +176,5 @@
                  }
              }]
          }]
-     }],
+     }]
-     "info": {
+ }
-         "url": "url",
-         "count": 8,
-         "first_time": "2017-12-25T19:46:01Z",
-         "last_time": "2017-12-25T19:47:05Z"
-     }
- }

backend/app/store/service/title.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
-	"github.com/go-pkgz/lcw"
+	"github.com/go-pkgz/lcw/v2"
 	log "github.com/go-pkgz/lgr"
 	"golang.org/x/net/html"
 )
@@ -19,32 +20,53 @@ const (
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
-	client http.Client
+	client         http.Client
-	cache  lcw.LoadingCache
+	cache          lcw.LoadingCache[string]
+	allowedDomains []string
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
-func NewTitleExtractor(client http.Client) *TitleExtractor {
+func NewTitleExtractor(client http.Client, allowedDomains []string) *TitleExtractor {
+	log.Printf("[DEBUG] creating extractor, allowed domains %+v", allowedDomains)
 	res := TitleExtractor{
-		client: client,
+		client:         client,
+		allowedDomains: allowedDomains,
 	}
 	var err error
-	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
+	o := lcw.NewOpts[string]()
+	res.cache, err = lcw.NewExpirableCache(o.TTL(teCacheTTL), o.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
 		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
-		res.cache = &lcw.Nop{}
+		res.cache = &lcw.Nop[string]{}
 	}
 	return &res
 }
 
 // Get page for url and return title
-func (t *TitleExtractor) Get(url string) (string, error) {
+func (t *TitleExtractor) Get(pageURL string) (string, error) {
+	// parse domain of the URL and check if it's in the allowed list
+	u, err := url.Parse(pageURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse url %s: %w", pageURL, err)
+	}
+	allowed := false
+	for _, domain := range t.allowedDomains {
+		if u.Hostname() == domain ||
+			(strings.HasSuffix(u.Hostname(), domain) && // suffix match, e.g. "example.com" matches "www.example.com"
+				u.Hostname()[len(u.Hostname())-len(domain)-1] == '.') { // but we should not match "notexample.com"
+			allowed = true
+			break
+		}
+	}
+	if !allowed {
+		return "", fmt.Errorf("domain %s is not allowed", u.Host)
+	}
 	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
 	defer client.CloseIdleConnections()
-	b, err := t.cache.Get(url, func() (interface{}, error) {
+	b, err := t.cache.Get(pageURL, func() (string, error) {
-		resp, err := client.Get(url)
+		resp, e := client.Get(pageURL)
-		if err != nil {
+		if e != nil {
-			return nil, fmt.Errorf("failed to load page %s: %w", url, err)
+			return "", fmt.Errorf("failed to load page %s: %w", pageURL, e)
 		}
 		defer func() {
 			if err = resp.Body.Close(); err != nil {
@@ -52,23 +74,23 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 			}
 		}()
 		if resp.StatusCode != 200 {
-			return nil, fmt.Errorf("can't load page %s, code %d", url, resp.StatusCode)
+			return "", fmt.Errorf("can't load page %s, code %d", pageURL, resp.StatusCode)
 		}
 
 		title, ok := t.getTitle(resp.Body)
 		if !ok {
-			return nil, fmt.Errorf("can't get title for %s", url)
+			return "", fmt.Errorf("can't get title for %s", pageURL)
 		}
 		return title, nil
 	})
 
 	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
-		_, _ = t.cache.Get(url, func() (interface{}, error) { return "", nil })
+		_, _ = t.cache.Get(pageURL, func() (string, error) { return "", nil })
 		return "", err
 	}
 
-	return b.(string), nil
+	return b, nil
 }
 
 // Close title extractor

backend/app/store/service/title_test.go

@@ -28,7 +28,7 @@ func TestTitle_GetTitle(t *testing.T) {
 		{`<html><body> 2222</body></html>`, false, ""},
 	}
 
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})
 	defer ex.Close()
 	for i, tt := range tbl {
 		tt := tt
@@ -41,7 +41,7 @@ func TestTitle_GetTitle(t *testing.T) {
 }
 
 func TestTitle_Get(t *testing.T) {
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -75,7 +75,7 @@ func TestTitle_GetConcurrent(t *testing.T) {
 	for n := 0; n < 1000; n++ {
 		body += "something something blah blah\n"
 	}
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -104,10 +104,10 @@ func TestTitle_GetConcurrent(t *testing.T) {
 }
 
 func TestTitle_GetFailed(t *testing.T) {
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
 	defer ex.Close()
 	var hits int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		atomic.AddInt32(&hits, 1)
 		w.WriteHeader(404)
 	}))
@@ -125,8 +125,8 @@ func TestTitle_GetFailed(t *testing.T) {
 }
 
 func TestTitle_DoubleClosed(t *testing.T) {
-	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{})
-	ex.Close()
+	assert.NoError(t, ex.Close())
 	// second call should not result in panic
-	ex.Close()
+	assert.NoError(t, ex.Close())
 }

backend/app/store/service/tree.go

@@ -10,8 +10,7 @@ import (
 
 // Tree is formatter making tree from the list of comments
 type Tree struct {
-	Nodes []*Node        `json:"comments"`
+	Nodes []*Node `json:"comments"`
-	Info  store.PostInfo `json:"info,omitempty"`
 }
 
 // Node is a comment with optional replies
@@ -30,22 +29,12 @@ type recurData struct {
 }
 
 // MakeTree gets unsorted list of comments and produces Tree
-// It will make store.PostInfo by itself and will mark Info.ReadOnly based on passed readOnlyAge
+func MakeTree(comments []store.Comment, sortType string) *Tree {
-// Tree maker is local and has no access to the data store. By this reason it has to make Info and won't be able
-// to handle store's read-only status. This status should be set by caller.
-func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree {
 	if len(comments) == 0 {
 		return &Tree{}
 	}
 
-	res := Tree{
+	res := Tree{}
-		Info: store.PostInfo{
-			URL:     comments[0].Locator.URL,
-			FirstTS: comments[0].Timestamp,
-			LastTS:  comments[0].Timestamp,
-		},
-	}
-	res.Info.Count = len(res.filter(comments, func(c store.Comment) bool { return !c.Deleted }))
 
 	topComments := res.filter(comments, func(c store.Comment) bool { return c.ParentID == "" })
 
@@ -54,23 +43,12 @@ func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree
 		node := Node{Comment: rootComment}
 
 		rd := recurData{}
-		commentsTree, tsModified, tsCreated := res.proc(comments, &node, &rd, rootComment.ID)
+		commentsTree := res.proc(comments, &node, &rd, rootComment.ID)
-		// skip deleted with no sub-comments ar all sub-comments deleted
+		// skip deleted with no sub-comments and all sub-comments deleted
 		if rootComment.Deleted && (len(commentsTree.Replies) == 0 || !rd.visible) {
 			continue
 		}
 
-		commentsTree.tsModified, commentsTree.tsCreated = tsModified, tsCreated
-		if commentsTree.tsCreated.Before(res.Info.FirstTS) {
-			res.Info.FirstTS = commentsTree.tsCreated
-		}
-		if commentsTree.tsModified.After(res.Info.LastTS) {
-			res.Info.LastTS = commentsTree.tsModified
-		}
-
-		res.Info.ReadOnly = readOnlyAge > 0 && !res.Info.FirstTS.IsZero() &&
-			res.Info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-
 		res.Nodes = append(res.Nodes, commentsTree)
 	}
 
@@ -79,7 +57,7 @@ func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree
 }
 
 // proc makes tree for one top-level comment recursively
-func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (result *Node, modified, created time.Time) {
+func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (result *Node) {
 	if rd.tsModified.IsZero() || rd.tsCreated.IsZero() {
 		rd.tsModified, rd.tsCreated = node.Comment.Timestamp, node.Comment.Timestamp
 	}
@@ -106,7 +84,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 	sort.Slice(node.Replies, func(i, j int) bool {
 		return node.Replies[i].Comment.Timestamp.Before(node.Replies[j].Comment.Timestamp)
 	})
-	return node, rd.tsModified, rd.tsCreated
+	node.tsModified, node.tsCreated = rd.tsModified, rd.tsCreated
+	return node
 }
 
 // filter returns comments for parentID

backend/app/store/service/tree_test.go

@@ -37,19 +37,15 @@ func TestMakeTree(t *testing.T) {
 		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
 	}
 
-	res := MakeTree(comments, "time", 0)
+	res := MakeTree(comments, "time")
 	resJSON, err := json.Marshal(&res)
 	require.NoError(t, err)
 
 	expJSON := mustLoadJSONFile(t, "testdata/tree.json")
 	assert.Equal(t, expJSON, resJSON)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
 
-	res = MakeTree([]store.Comment{}, "time", 0)
+	res = MakeTree([]store.Comment{}, "time")
 	assert.Equal(t, &Tree{}, res)
-
-	res = MakeTree(comments, "time", 10)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
 }
 
 func TestMakeEmptySubtree(t *testing.T) {
@@ -79,7 +75,7 @@ func TestMakeEmptySubtree(t *testing.T) {
 		{Locator: loc, ID: "3", Timestamp: ts(48, 1), Deleted: true}, // deleted top level
 	}
 
-	res := MakeTree(comments, "time", 0)
+	res := MakeTree(comments, "time")
 	resJSON, err := json.Marshal(&res)
 	require.NoError(t, err)
 	t.Log(string(resJSON))
@@ -108,50 +104,50 @@ func TestTreeSortNodes(t *testing.T) {
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
 	}
 
-	res := MakeTree(comments, "+active", 0)
+	res := MakeTree(comments, "+active")
 	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 
-	res = MakeTree(comments, "-active", 0)
+	res = MakeTree(comments, "-active")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "+time", 0)
+	res = MakeTree(comments, "+time")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "-time", 0)
+	res = MakeTree(comments, "-time")
 	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "score", 0)
+	res = MakeTree(comments, "score")
 	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "+score", 0)
+	res = MakeTree(comments, "+score")
 	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
 
-	res = MakeTree(comments, "-score", 0)
+	res = MakeTree(comments, "-score")
 	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "+controversy", 0)
+	res = MakeTree(comments, "+controversy")
 	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
 	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
 
-	res = MakeTree(comments, "-controversy", 0)
+	res = MakeTree(comments, "-controversy")
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
 	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
 
-	res = MakeTree(comments, "undefined", 0)
+	res = MakeTree(comments, "undefined")
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
 }
@@ -164,7 +160,7 @@ func BenchmarkTree(b *testing.B) {
 	assert.NoError(b, err)
 
 	for i := 0; i < b.N; i++ {
-		res := MakeTree(comments, "time", 0)
+		res := MakeTree(comments, "time")
 		assert.NotNil(b, res)
 	}
 }

backend/go.mod

@@ -1,78 +1,78 @@
 module github.com/umputun/remark42/backend
 
-go 1.17
+go 1.21
 
 require (
 	github.com/Depado/bfchroma/v2 v2.0.0
-	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/PuerkitoBio/goquery v1.9.1
-	github.com/alecthomas/chroma/v2 v2.3.0
+	github.com/alecthomas/chroma/v2 v2.13.0
 	github.com/didip/tollbooth/v7 v7.0.1
 	github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f
-	github.com/go-chi/chi/v5 v5.0.7
+	github.com/go-chi/chi/v5 v5.0.12
 	github.com/go-chi/cors v1.2.1
-	github.com/go-chi/render v1.0.2
+	github.com/go-chi/render v1.0.3
-	github.com/go-pkgz/auth v1.20.1-0.20230103203948-168bd5a101b7
+	github.com/go-pkgz/auth v1.23.0
 	github.com/go-pkgz/jrpc v0.3.0
-	github.com/go-pkgz/lcw v1.0.3-0.20221226231215-a66ea7c4aff7
+	github.com/go-pkgz/lcw/v2 v2.0.0
-	github.com/go-pkgz/lgr v0.10.4
+	github.com/go-pkgz/lgr v0.11.1
-	github.com/go-pkgz/notify v0.2.0
+	github.com/go-pkgz/notify v1.1.0
 	github.com/go-pkgz/repeater v1.1.3
-	github.com/go-pkgz/rest v1.16.0
+	github.com/go-pkgz/rest v1.19.0
-	github.com/go-pkgz/syncs v1.2.0
+	github.com/go-pkgz/syncs v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
-	github.com/gorilla/feeds v1.1.1
+	github.com/gorilla/feeds v1.1.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/jessevdk/go-flags v1.5.0
-	github.com/kyokomi/emoji/v2 v2.2.10
+	github.com/kyokomi/emoji/v2 v2.2.12
-	github.com/microcosm-cc/bluemonday v1.0.20
+	github.com/microcosm-cc/bluemonday v1.0.26
-	github.com/rs/xid v1.4.0
+	github.com/rs/xid v1.5.0
 	github.com/russross/blackfriday/v2 v2.1.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.9.0
-	go.etcd.io/bbolt v1.3.6
+	go.etcd.io/bbolt v1.3.9
-	go.uber.org/goleak v1.2.0
+	go.uber.org/goleak v1.3.0
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
+	golang.org/x/crypto v0.22.0
-	golang.org/x/image v0.0.0-20220617043117-41969df76e82
+	golang.org/x/image v0.15.0
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
+	golang.org/x/net v0.24.0
 )
 
 require (
-	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go/compute v1.25.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/ajg/form v1.5.1 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dghubble/oauth1 v0.7.1 // indirect
+	github.com/dghubble/oauth1 v0.7.3 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
-	github.com/go-oauth2/oauth2/v4 v4.5.1 // indirect
+	github.com/go-oauth2/oauth2/v4 v4.5.2 // indirect
-	github.com/go-pkgz/email v0.4.1 // indirect
+	github.com/go-pkgz/email v0.5.0 // indirect
-	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
+	github.com/go-pkgz/expirable-cache v1.0.0 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/golang-lru v0.6.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/klauspost/compress v1.15.2 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/slack-go/slack v0.11.3 // indirect
+	github.com/redis/go-redis/v9 v9.4.0 // indirect
+	github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d // indirect
+	github.com/slack-go/slack v0.12.4 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
-	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.mongodb.org/mongo-driver v1.10.2 // indirect
+	go.mongodb.org/mongo-driver v1.14.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

backend/remark.rest

@@ -103,6 +103,14 @@ GET {{host}}/api/v1/admin/blocked?site={{site}}
 ### delete comment by id
 DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
 
+### export site (for backup)
+GET {{host}}/api/v1/admin/export?site={{site}}&mode=stream
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+
+### export site (for backup) to .gz file
+GET {{host}}/api/v1/admin/export?site={{site}}&mode=file
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE5NzYwNTY3NTYsImp0aSI6IjJlOGJmMTE5OTI0MjQxMDRjYjFhZGRlODllMWYwNGFiMTg4YWZjMzQiLCJpYXQiOjE1NzYwNTY0NTYsImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.6Qt5s2enBMRC-Jmsua01yViVYI95Dx6BPBMaNjj36d4
+
 ### get post info
 GET {{host}}/api/v1/info?site={{site}}&url={{url}}
 

backend/vendor/cloud.google.com/go/compute/internal/version.go

@@ -0,0 +1,18 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+// Version is the current tagged release of the library.
+const Version = "1.25.1"

backend/vendor/cloud.google.com/go/compute/metadata/CHANGES.md

@@ -0,0 +1,19 @@
+# Changes
+
+## [0.2.3](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.2...compute/metadata/v0.2.3) (2022-12-15)
+
+
+### Bug Fixes
+
+* **compute/metadata:** Switch DNS lookup to an absolute lookup ([119b410](https://github.com/googleapis/google-cloud-go/commit/119b41060c7895e45e48aee5621ad35607c4d021)), refs [#7165](https://github.com/googleapis/google-cloud-go/issues/7165)
+
+## [0.2.2](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.1...compute/metadata/v0.2.2) (2022-12-01)
+
+
+### Bug Fixes
+
+* **compute/metadata:** Set IdleConnTimeout for http.Client ([#7084](https://github.com/googleapis/google-cloud-go/issues/7084)) ([766516a](https://github.com/googleapis/google-cloud-go/commit/766516aaf3816bfb3159efeea65aa3d1d205a3e2)), refs [#5430](https://github.com/googleapis/google-cloud-go/issues/5430)
+
+## [0.1.0] (2022-10-26)
+
+Initial release of metadata being it's own module.

backend/vendor/cloud.google.com/go/compute/metadata/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

backend/vendor/cloud.google.com/go/compute/metadata/README.md

@@ -0,0 +1,27 @@
+# Compute API
+
+[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata)
+
+This is a utility library for communicating with Google Cloud metadata service
+on Google Cloud.
+
+## Install
+
+```bash
+go get cloud.google.com/go/compute/metadata
+```
+
+## Go Version Support
+
+See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported)
+section in the root directory's README.
+
+## Contributing
+
+Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md)
+document for details.
+
+Please note that this project is released with a Contributor Code of Conduct.
+By participating in this project you agree to abide by its terms. See
+[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct)
+for more information.

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -16,7 +16,7 @@
 // metadata and API service accounts.
 //
 // This package is a wrapper around the GCE metadata service,
-// as documented at https://developers.google.com/compute/docs/metadata.
+// as documented at https://cloud.google.com/compute/docs/metadata/overview.
 package metadata // import "cloud.google.com/go/compute/metadata"
 
 import (
@@ -70,7 +70,9 @@ func newDefaultHTTPClient() *http.Client {
 				Timeout:   2 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).Dial,
+			IdleConnTimeout: 60 * time.Second,
 		},
+		Timeout: 5 * time.Second,
 	}
 }
 
@@ -145,7 +147,7 @@ func testOnGCE() bool {
 
 	go func() {
 		resolver := &net.Resolver{}
-		addrs, err := resolver.LookupHost(ctx, "metadata.google.internal")
+		addrs, err := resolver.LookupHost(ctx, "metadata.google.internal.")
 		if err != nil || len(addrs) == 0 {
 			resc <- false
 			return

backend/vendor/cloud.google.com/go/compute/metadata/tidyfix.go

@@ -0,0 +1,23 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file, and the {{.RootMod}} import, won't actually become part of
+// the resultant binary.
+//go:build modhack
+// +build modhack
+
+package metadata
+
+// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository
+import _ "cloud.google.com/go/compute/internal"

backend/vendor/github.com/PuerkitoBio/goquery/README.md

@@ -22,7 +22,7 @@ Syntax-wise, it is as close as possible to jQuery, with the same function names
 
 ## Installation
 
-Please note that because of the net/html dependency, goquery requires Go1.1+ and is tested on Go1.7+.
+Please note that starting with version `v1.9.0` of goquery, Go 1.18+ is required due to the use of generics. For previous goquery versions, a Go version of 1.1+ was required because of the `net/html` dependency. Ongoing goquery development is tested on the latest 2 versions of Go.
 
     $ go get github.com/PuerkitoBio/goquery
 
@@ -40,6 +40,9 @@ Please note that because of the net/html dependency, goquery requires Go1.1+ and
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2024-02-29 (v1.9.1)** : Improve allocation and performance of the `Map` function and `Selection.Map` method, better document the cascadia differences (thanks [@jwilsson](https://github.com/jwilsson)).
+*    **2024-02-22 (v1.9.0)** : Add a generic `Map` function, **goquery now requires Go version 1.18+** (thanks [@Fesaa](https://github.com/Fesaa)).
+*    **2023-02-18 (v1.8.1)** : Update `go.mod` dependencies, update CI workflow.
 *    **2021-10-25 (v1.8.0)** : Add `Render` function to render a `Selection` to an `io.Writer` (thanks [@anthonygedeon](https://github.com/anthonygedeon)).
 *    **2021-07-11 (v1.7.1)** : Update go.mod dependencies and add dependabot config (thanks [@jauderho](https://github.com/jauderho)).
 *    **2021-06-14 (v1.7.0)** : Add `Single` and `SingleMatcher` functions to optimize first-match selection (thanks [@gdollardollar](https://github.com/gdollardollar)).
@@ -87,7 +90,7 @@ Utility functions that are not in jQuery but are useful in Go are implemented as
 
 The complete [package reference documentation can be found here][doc].
 
-Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string):
+Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Also, the selectors work more like the DOM's `querySelectorAll`, than jQuery's matchers - they have no concept of contextual matching (for some concrete examples of what that means, see [this ticket](https://github.com/andybalholm/cascadia/issues/61)). In practice, it doesn't matter very often but it's something worth mentioning. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string):
 
 * `Find("~")` returns an empty selection because the selector string doesn't match anything.
 * `Add("~")` returns a new selection that holds the same nodes as the original selection, because it didn't add any node (selector string didn't match anything).
@@ -154,6 +157,9 @@ func main() {
 - [Geziyor](https://github.com/geziyor/geziyor), a fast web crawling & scraping framework for Go. Supports JS rendering.
 - [Pagser](https://github.com/foolin/pagser), a simple, easy, extensible, configurable HTML parser to struct based on goquery and struct tags.
 - [stitcherd](https://github.com/vhodges/stitcherd), A server for doing server side includes using css selectors and DOM updates.
+- [goskyr](https://github.com/jakopako/goskyr), an easily configurable command-line scraper written in Go.
+- [goGetJS](https://github.com/davemolk/goGetJS), a tool for extracting, searching, and saving JavaScript files (with optional headless browser).
+- [fitter](https://github.com/PxyUp/fitter), a tool for selecting values from JSON, XML, HTML and XPath formatted pages.
 
 ## Support
 
@@ -162,7 +168,7 @@ There are a number of ways you can support the project:
 * Use it, star it, build something with it, spread the word!
   - If you do build something open-source or otherwise publicly-visible, let me know so I can add it to the [Related Projects](#related-projects) section!
 * Raise issues to improve the project (note: doc typos and clarifications are issues too!)
-  - Please search existing issues before opening a new one - it may have already been adressed.
+  - Please search existing issues before opening a new one - it may have already been addressed.
 * Pull requests: please discuss new code in an issue first, unless the fix is really trivial.
   - Make sure new code is tested.
   - Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue.

backend/vendor/github.com/PuerkitoBio/goquery/iteration.go

@@ -31,8 +31,16 @@ func (s *Selection) EachWithBreak(f func(int, *Selection) bool) *Selection {
 // element in that selection starting at 0, and a *Selection that contains
 // only that element.
 func (s *Selection) Map(f func(int, *Selection) string) (result []string) {
+	return Map(s, f)
+}
+
+// Map is the generic version of Selection.Map, allowing any type to be
+// returned.
+func Map[E any](s *Selection, f func(int, *Selection) E) (result []E) {
+	result = make([]E, len(s.Nodes))
+
 	for i, n := range s.Nodes {
-		result = append(result, f(i, newSingleSelection(n, s.document)))
+		result[i] = f(i, newSingleSelection(n, s.document))
 	}
 
 	return result

backend/vendor/github.com/PuerkitoBio/goquery/traversal.go

@@ -20,6 +20,12 @@ const (
 // Find gets the descendants of each element in the current set of matched
 // elements, filtered by a selector. It returns a new Selection object
 // containing these matched elements.
+//
+// Note that as for all methods accepting a selector string, the selector is
+// compiled and applied by the cascadia package and inherits its behavior and
+// constraints regarding supported selectors. See the note on cascadia in
+// the goquery documentation here:
+// https://github.com/PuerkitoBio/goquery?tab=readme-ov-file#api
 func (s *Selection) Find(selector string) *Selection {
 	return pushStack(s, findWithMatcher(s.Nodes, compileMatcher(selector)))
 }

backend/vendor/github.com/PuerkitoBio/goquery/utilities.go

@@ -58,8 +58,8 @@ func nodeName(node *html.Node) string {
 	}
 }
 
-// Render renders the html of the first element from selector and writes it to
+// Render renders the HTML of the first item in the selection and writes it to
-// the writer.  It behaves the same as OuterHtml but writes to w instead of
+// the writer. It behaves the same as OuterHtml but writes to w instead of
 // returning the string.
 func Render(w io.Writer, s *Selection) error {
 	if s.Length() == 0 {
@@ -73,7 +73,7 @@ func Render(w io.Writer, s *Selection) error {
 // the selection - that is, the HTML including the first element's
 // tag and attributes.
 //
-// Unlike InnerHtml, this is a function and not a method on the Selection,
+// Unlike Html, this is a function and not a method on the Selection,
 // because this is not a jQuery method (in javascript-land, this is
 // a property provided by the DOM).
 func OuterHtml(s *Selection) (string, error) {

backend/vendor/github.com/alecthomas/chroma/v2/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+indent_style = tab
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.xml]
+indent_style = space
+indent_size = 2
+insert_final_newline = false
+
+[*.yml]
+indent_style = space
+indent_size = 2

backend/vendor/github.com/alecthomas/chroma/v2/.gitignore

@@ -1,4 +1,8 @@
 # Binaries for programs and plugins
+.git
+.idea
+.vscode
+.hermit
 *.exe
 *.dll
 *.so
@@ -17,3 +21,5 @@
 _models/
 
 _examples/
+*.min.*
+build/

backend/vendor/github.com/alecthomas/chroma/v2/.golangci.yml

@@ -37,6 +37,20 @@ linters:
     - wrapcheck
     - stylecheck
     - thelper
+    - nonamedreturns
+    - revive
+    - dupword
+    - exhaustruct
+    - varnamelen
+    - forcetypeassert
+    - ireturn
+    - maintidx
+    - govet
+    - nosnakecase
+    - testableexamples
+    - musttag
+    - depguard
+    - goconst
 
 linters-settings:
   govet:

backend/vendor/github.com/alecthomas/chroma/v2/Bitfile

@@ -0,0 +1,24 @@
+VERSION = %(git describe --tags --dirty  --always)%
+export CGOENABLED = 0
+
+tokentype_enumer.go: types.go
+  build: go generate
+
+# Regenerate the list of lexers in the README
+README.md: lexers/*.go lexers/*/*.xml table.py
+  build: ./table.py
+  -clean
+
+implicit %{1}%{2}.min.%{3}: **/*.{css,js}
+  build: esbuild --bundle %{IN} --minify --outfile=%{OUT}
+
+implicit build/%{1}: cmd/*
+  cd cmd/%{1}
+  inputs: cmd/%{1}/**/* **/*.go
+  build: go build -ldflags="-X 'main.version=%{VERSION}'" -o ../../build/%{1} .
+
+#upload: chromad
+#  build:
+#    scp chromad root@swapoff.org:
+#    ssh root@swapoff.org 'install -m755 ./chromad /srv/http/swapoff.org/bin && service chromad restart'
+#    touch upload

backend/vendor/github.com/alecthomas/chroma/v2/Makefile

@@ -1,6 +1,8 @@
 .PHONY: chromad upload all
 
 VERSION ?= $(shell git describe --tags --dirty  --always)
+export GOOS ?= linux
+export GOARCH ?= amd64
 
 all: README.md tokentype_string.go
 
@@ -11,9 +13,11 @@ tokentype_string.go: types.go
 	go generate
 
 chromad:
-	rm -f chromad
+	rm -rf build
-	(export CGOENABLED=0 GOOS=linux GOARCH=amd64; cd ./cmd/chromad && go build -ldflags="-X 'main.version=$(VERSION)'" -o ../../chromad .)
+	esbuild --bundle cmd/chromad/static/index.js --minify --outfile=cmd/chromad/static/index.min.js
+	esbuild --bundle cmd/chromad/static/index.css --minify --outfile=cmd/chromad/static/index.min.css
+	(export CGOENABLED=0 ; cd ./cmd/chromad && go build -ldflags="-X 'main.version=$(VERSION)'" -o ../../build/chromad .)
 
-upload: chromad
+upload: build/chromad
-	scp chromad root@swapoff.org: && \
+	scp build/chromad root@swapoff.org: && \
 		ssh root@swapoff.org 'install -m755 ./chromad /srv/http/swapoff.org/bin && service chromad restart'

backend/vendor/github.com/alecthomas/chroma/v2/README.md

@@ -1,7 +1,6 @@
 # Chroma — A general purpose syntax highlighter in pure Go
-[![Golang Documentation](https://godoc.org/github.com/alecthomas/chroma?status.svg)](https://godoc.org/github.com/alecthomas/chroma) [![CI](https://github.com/alecthomas/chroma/actions/workflows/ci.yml/badge.svg)](https://github.com/alecthomas/chroma/actions/workflows/ci.yml) [![Slack chat](https://img.shields.io/static/v1?logo=slack&style=flat&label=slack&color=green&message=gophers)](https://invite.slack.golangbridge.org/)
 
-> **NOTE:** As Chroma has just been released, its API is still in flux. That said, the high-level interface should not change significantly.
+[![Golang Documentation](https://godoc.org/github.com/alecthomas/chroma?status.svg)](https://godoc.org/github.com/alecthomas/chroma) [![CI](https://github.com/alecthomas/chroma/actions/workflows/ci.yml/badge.svg)](https://github.com/alecthomas/chroma/actions/workflows/ci.yml) [![Slack chat](https://img.shields.io/static/v1?logo=slack&style=flat&label=slack&color=green&message=gophers)](https://invite.slack.golangbridge.org/)
 
 Chroma takes source code and other structured text and converts it into syntax
 highlighted HTML, ANSI-coloured text, etc.
@@ -9,71 +8,72 @@ highlighted HTML, ANSI-coloured text, etc.
 Chroma is based heavily on [Pygments](http://pygments.org/), and includes
 translators for Pygments lexers and styles.
 
-<a id="markdown-table-of-contents" name="table-of-contents"></a>
 ## Table of Contents
 
 <!-- TOC -->
 
-1. [Table of Contents](#table-of-contents)
+1. [Supported languages](#supported-languages)
-2. [Supported languages](#supported-languages)
+2. [Try it](#try-it)
-3. [Try it](#try-it)
+3. [Using the library](#using-the-library)
-4. [Using the library](#using-the-library)
+   1. [Quick start](#quick-start)
-    1. [Quick start](#quick-start)
+   2. [Identifying the language](#identifying-the-language)
-    2. [Identifying the language](#identifying-the-language)
+   3. [Formatting the output](#formatting-the-output)
-    3. [Formatting the output](#formatting-the-output)
+   4. [The HTML formatter](#the-html-formatter)
-    4. [The HTML formatter](#the-html-formatter)
+4. [More detail](#more-detail)
-5. [More detail](#more-detail)
+   1. [Lexers](#lexers)
-    1. [Lexers](#lexers)
+   2. [Formatters](#formatters)
-    2. [Formatters](#formatters)
+   3. [Styles](#styles)
-    3. [Styles](#styles)
+5. [Command-line interface](#command-line-interface)
-6. [Command-line interface](#command-line-interface)
+6. [Testing lexers](#testing-lexers)
 7. [What's missing compared to Pygments?](#whats-missing-compared-to-pygments)
 
 <!-- /TOC -->
 
-<a id="markdown-supported-languages" name="supported-languages"></a>
 ## Supported languages
 
-Prefix | Language
+| Prefix | Language                                                                                                                                                                                                                                            |
-:----: | --------
+| :----: | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-A | ABAP, ABNF, ActionScript, ActionScript 3, Ada, Angular2, ANTLR, ApacheConf, APL, AppleScript, Arduino, Awk
+|   A    | ABAP, ABNF, ActionScript, ActionScript 3, Ada, Agda, AL, Alloy, Angular2, ANTLR, ApacheConf, APL, AppleScript, ArangoDB AQL, Arduino, ArmAsm, AutoHotkey, AutoIt, Awk                                                                               |
-B | Ballerina, Base Makefile, Bash, Batchfile, BibTeX, Bicep, BlitzBasic, BNF, Brainfuck
+|   B    | Ballerina, Bash, Bash Session, Batchfile, BibTeX, Bicep, BlitzBasic, BNF, BQN, Brainfuck                                                                                                                                                            |
-C | C, C#, C++, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Crystal, CSS, Cython
+|   C    | C, C#, C++, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Crystal, CSS, Cython                                   |
-D | D, Dart, Diff, Django/Jinja, Docker, DTD, Dylan
+|   D    | D, Dart, Dax, Desktop Entry, Diff, Django/Jinja, dns, Docker, DTD, Dylan                                                                                                                                                                            |
-E | EBNF, Elixir, Elm, EmacsLisp, Erlang
+|   E    | EBNF, Elixir, Elm, EmacsLisp, Erlang                                                                                                                                                                                                                |
-F | Factor, Fish, Forth, Fortran, FSharp
+|   F    | Factor, Fennel, Fish, Forth, Fortran, FortranFixed, FSharp                                                                                                                                                                                          |
-G | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, Gherkin, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groff, Groovy
+|   G    | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, Gherkin, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groff, Groovy                                                                                                             |
-H | Handlebars, Haskell, Haxe, HCL, Hexdump, HLB, HTML, HTTP, Hy
+|   H    | Handlebars, Hare, Haskell, Haxe, HCL, Hexdump, HLB, HLSL, HolyC, HTML, HTTP, Hy                                                                                                                                                                     |
-I | Idris, Igor, INI, Io
+|   I    | Idris, Igor, INI, Io, ISCdhcpd                                                                                                                                                                                                                      |
-J | J, Java, JavaScript, JSON, Julia, Jungle
+|   J    | J, Java, JavaScript, JSON, Julia, Jungle                                                                                                                                                                                                            |
-K | Kotlin
+|   K    | Kotlin                                                                                                                                                                                                                                              |
-L | Lighttpd configuration file, LLVM, Lua
+|   L    | Lighttpd configuration file, LLVM, Lua                                                                                                                                                                                                              |
-M | Mako, markdown, Mason, Mathematica, Matlab, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL
+|   M    | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, mcfunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL                                                                          |
-N | NASM, Newspeak, Nginx configuration file, Nim, Nix
+|   N    | NASM, Natural, Newspeak, Nginx configuration file, Nim, Nix                                                                                                                                                                                         |
-O | Objective-C, OCaml, Octave, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode
+|   O    | Objective-C, OCaml, Octave, Odin, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode                                                                                                                                                                  |
-P | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerShell, Prolog, PromQL, Properties, Protocol Buffer, Puppet, Python 2, Python
+|   P    | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Plutus Core, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerQuery, PowerShell, Prolog, PromQL, Promela, properties, Protocol Buffer, PRQL, PSL, Puppet, Python, Python 2 |
-Q | QBasic
+|   Q    | QBasic, QML                                                                                                                                                                                                                                         |
-R | R, Racket, Ragel, Raku, react, ReasonML, reg, reStructuredText, Rexx, Ruby, Rust
+|   R    | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RPMSpec, Ruby, Rust                                                                                                                                                     |
-S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Smalltalk, Smarty, Snobol, Solidity, SPARQL, SQL, SquidConf, Standard ML, Stylus, Svelte, Swift, SYSTEMD, systemverilog
+|   S    | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog                                                |
-T | TableGen, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData
+|   T    | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData                                                     |
-V | VB.net, verilog, VHDL, VimL, vue
+|   V    | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue                                                                                                                                                                                             |
-W | WDTE
+|   W    | WDTE, WebGPU Shading Language, Whiley                                                                                                                                                                                                               |
-X | XML, Xorg
+|   X    | XML, Xorg                                                                                                                                                                                                                                           |
-Y | YAML, YANG
+|   Y    | YAML, YANG                                                                                                                                                                                                                                          |
-Z | Zig
+|   Z    | Z80 Assembly, Zed, Zig                                                                                                                                                                                                                              |
-
 
 _I will attempt to keep this section up to date, but an authoritative list can be
 displayed with `chroma --list`._
 
-<a id="markdown-try-it" name="try-it"></a>
 ## Try it
 
 Try out various languages and styles on the [Chroma Playground](https://swapoff.org/chroma/playground/).
 
-<a id="markdown-using-the-library" name="using-the-library"></a>
 ## Using the library
 
+This is version 2 of Chroma, use the import path:
+
+```go
+import "github.com/alecthomas/chroma/v2"
+```
+
 Chroma, like Pygments, has the concepts of
 [lexers](https://github.com/alecthomas/chroma/tree/master/lexers),
 [formatters](https://github.com/alecthomas/chroma/tree/master/formatters) and
@@ -92,7 +92,6 @@ In all cases, if a lexer, formatter or style can not be determined, `nil` will
 be returned. In this situation you may want to default to the `Fallback`
 value in each respective package, which provides sane defaults.
 
-<a id="markdown-quick-start" name="quick-start"></a>
 ### Quick start
 
 A convenience function exists that can be used to simply format some source
@@ -102,7 +101,6 @@ text, without any effort:
 err := quick.Highlight(os.Stdout, someSourceCode, "go", "html", "monokai")
 ```
 
-<a id="markdown-identifying-the-language" name="identifying-the-language"></a>
 ### Identifying the language
 
 To highlight code, you'll first have to identify what language the code is
@@ -110,21 +108,21 @@ written in. There are three primary ways to do that:
 
 1. Detect the language from its filename.
 
-    ```go
+   ```go
-    lexer := lexers.Match("foo.go")
+   lexer := lexers.Match("foo.go")
-    ```
+   ```
 
-3. Explicitly specify the language by its Chroma syntax ID (a full list is available from `lexers.Names()`).
+2. Explicitly specify the language by its Chroma syntax ID (a full list is available from `lexers.Names()`).
 
-    ```go
+   ```go
-    lexer := lexers.Get("go")
+   lexer := lexers.Get("go")
-    ```
+   ```
 
 3. Detect the language from its content.
 
-    ```go
+   ```go
-    lexer := lexers.Analyse("package main\n\nfunc main()\n{\n}\n")
+   lexer := lexers.Analyse("package main\n\nfunc main()\n{\n}\n")
-    ```
+   ```
 
 In all cases, `nil` will be returned if the language can not be identified.
 
@@ -142,7 +140,6 @@ token types into a single token:
 lexer = chroma.Coalesce(lexer)
 ```
 
-<a id="markdown-formatting-the-output" name="formatting-the-output"></a>
 ### Formatting the output
 
 Once a language is identified you will need to pick a formatter and a style (theme).
@@ -171,7 +168,6 @@ And finally, format the tokens from the iterator:
 err := formatter.Format(w, style, iterator)
 ```
 
-<a id="markdown-the-html-formatter" name="the-html-formatter"></a>
 ### The HTML formatter
 
 By default the `html` registered formatter generates standalone HTML with
@@ -185,7 +181,7 @@ following constructor options:
 - `ClassPrefix(prefix)` - prefix each generated CSS class.
 - `TabWidth(width)` - Set the rendered tab width, in characters.
 - `WithLineNumbers()` - Render line numbers (style with `LineNumbers`).
-- `LinkableLineNumbers()` - Make the line numbers linkable and be a link to themselves.
+- `WithLinkableLineNumbers()` - Make the line numbers linkable and be a link to themselves.
 - `HighlightLines(ranges)` - Highlight lines in these ranges (style with `LineHighlight`).
 - `LineNumbersInTable()` - Use a table for formatting line numbers and code, rather than spans.
 
@@ -196,10 +192,8 @@ formatter := html.New(html.WithClasses(true))
 err := formatter.WriteCSS(w, style)
 ```
 
-<a id="markdown-more-detail" name="more-detail"></a>
 ## More detail
 
-<a id="markdown-lexers" name="lexers"></a>
 ### Lexers
 
 See the [Pygments documentation](http://pygments.org/docs/lexerdevelopment/)
@@ -207,20 +201,18 @@ for details on implementing lexers. Most concepts apply directly to Chroma,
 but see existing lexer implementations for real examples.
 
 In many cases lexers can be automatically converted directly from Pygments by
-using the included Python 3 script `pygments2chroma.py`. I use something like
+using the included Python 3 script `pygments2chroma_xml.py`. I use something like
 the following:
 
 ```sh
-python3 _tools/pygments2chroma.py \
+python3 _tools/pygments2chroma_xml.py \
   pygments.lexers.jvm.KotlinLexer \
-  > lexers/k/kotlin.go \
+  > lexers/embedded/kotlin.xml
-  && gofmt -s -w lexers/k/kotlin.go
 ```
 
 See notes in [pygments-lexers.txt](https://github.com/alecthomas/chroma/blob/master/pygments-lexers.txt)
 for a list of lexers, and notes on some of the issues importing them.
 
-<a id="markdown-formatters" name="formatters"></a>
 ### Formatters
 
 Chroma supports HTML output, as well as terminal output in 8 colour, 256 colour, and true-colour.
@@ -228,26 +220,29 @@ Chroma supports HTML output, as well as terminal output in 8 colour, 256 colour,
 A `noop` formatter is included that outputs the token text only, and a `tokens`
 formatter outputs raw tokens. The latter is useful for debugging lexers.
 
-<a id="markdown-styles" name="styles"></a>
 ### Styles
 
-Chroma styles use the [same syntax](http://pygments.org/docs/styles/) as Pygments.
+Chroma styles are defined in XML. The style entries use the
+[same syntax](http://pygments.org/docs/styles/) as Pygments.
 
-All Pygments styles have been converted to Chroma using the `_tools/style.py` script.
+All Pygments styles have been converted to Chroma using the `_tools/style.py`
+script.
 
-When you work with one of [Chroma's styles](https://github.com/alecthomas/chroma/tree/master/styles), know that the `chroma.Background` token type provides the default style for tokens. It does so by defining a foreground color and background color.
+When you work with one of [Chroma's styles](https://github.com/alecthomas/chroma/tree/master/styles),
+know that the `Background` token type provides the default style for tokens. It does so
+by defining a foreground color and background color.
 
-For example, this gives each token name not defined in the style a default color of `#f8f8f8` and uses `#000000` for the highlighted code block's background:
+For example, this gives each token name not defined in the style a default color
+of `#f8f8f8` and uses `#000000` for the highlighted code block's background:
 
-~~~go
+```xml
-chroma.Background: "#f8f8f2 bg:#000000",
+<entry type="Background" style="#f8f8f2 bg:#000000"/>
-~~~
+```
 
 Also, token types in a style file are hierarchical. For instance, when `CommentSpecial` is not defined, Chroma uses the token style from `Comment`. So when several comment tokens use the same color, you'll only need to define `Comment` and override the one that has a different color.
 
 For a quick overview of the available styles and how they look, check out the [Chroma Style Gallery](https://xyproto.github.io/splash/docs/).
 
-<a id="markdown-command-line-interface" name="command-line-interface"></a>
 ## Command-line interface
 
 A command-line interface to Chroma is included.
@@ -272,14 +267,31 @@ on under the hood for easy integration with [lesspipe shipping with
 Debian and derivatives](https://manpages.debian.org/lesspipe#USER_DEFINED_FILTERS);
 for that setup the `chroma` executable can be just symlinked to `~/.lessfilter`.
 
-<a id="markdown-whats-missing-compared-to-pygments" name="whats-missing-compared-to-pygments"></a>
+## Testing lexers
+
+If you edit some lexers and want to try it, open a shell in `cmd/chromad` and run:
+
+```shell
+go run .
+```
+
+A Link will be printed. Open it in your Browser. Now you can test on the Playground with your local changes.
+
+If you want to run the tests and the lexers, open a shell in the root directory and run:
+
+```shell
+go test ./lexers
+```
+
+When updating or adding a lexer, please add tests. See [lexers/README.md](lexers/README.md) for more.
+
 ## What's missing compared to Pygments?
 
 - Quite a few lexers, for various reasons (pull-requests welcome):
-    - Pygments lexers for complex languages often include custom code to
+  - Pygments lexers for complex languages often include custom code to
-      handle certain aspects, such as Raku's ability to nest code inside
+    handle certain aspects, such as Raku's ability to nest code inside
-      regular expressions. These require time and effort to convert.
+    regular expressions. These require time and effort to convert.
-    - I mostly only converted languages I had heard of, to reduce the porting cost.
+  - I mostly only converted languages I had heard of, to reduce the porting cost.
 - Some more esoteric features of Pygments are omitted for simplicity.
 - Though the Chroma API supports content detection, very few languages support them.
   I have plans to implement a statistical analyser at some point, but not enough time.

backend/vendor/github.com/alecthomas/chroma/v2/emitters.go

@@ -81,42 +81,39 @@ func ByGroupNames(emitters map[string]Emitter) Emitter {
 }
 
 // UsingByGroup emits tokens for the matched groups in the regex using a
-// "sublexer". Used when lexing code blocks where the name of a sublexer is
+// sublexer. Used when lexing code blocks where the name of a sublexer is
 // contained within the block, for example on a Markdown text block or SQL
 // language block.
 //
-// The sublexer will be retrieved using sublexerGetFunc (typically
+// An attempt to load the sublexer will be made using the captured value from
-// internal.Get), using the captured value from the matched sublexerNameGroup.
+// the text of the matched sublexerNameGroup. If a sublexer matching the
-//
+// sublexerNameGroup is available, then tokens for the matched codeGroup will
-// If sublexerGetFunc returns a non-nil lexer for the captured sublexerNameGroup,
+// be emitted using the sublexer. Otherwise, if no sublexer is available, then
-// then tokens for the matched codeGroup will be emitted using the retrieved
+// tokens will be emitted from the passed emitter.
-// lexer. Otherwise, if the sublexer is nil, then tokens will be emitted from
-// the passed emitter.
 //
 // Example:
 //
-// 	var Markdown = internal.Register(MustNewLexer(
+//	var Markdown = internal.Register(MustNewLexer(
-// 		&Config{
+//		&Config{
-// 			Name:      "markdown",
+//			Name:      "markdown",
-// 			Aliases:   []string{"md", "mkd"},
+//			Aliases:   []string{"md", "mkd"},
-// 			Filenames: []string{"*.md", "*.mkd", "*.markdown"},
+//			Filenames: []string{"*.md", "*.mkd", "*.markdown"},
-// 			MimeTypes: []string{"text/x-markdown"},
+//			MimeTypes: []string{"text/x-markdown"},
-// 		},
+//		},
-// 		Rules{
+//		Rules{
-// 			"root": {
+//			"root": {
-// 				{"^(```)(\\w+)(\\n)([\\w\\W]*?)(^```$)",
+//				{"^(```)(\\w+)(\\n)([\\w\\W]*?)(^```$)",
-// 					UsingByGroup(
+//					UsingByGroup(
-// 						internal.Get,
+//						2, 4,
-// 						2, 4,
+//						String, String, String, Text, String,
-// 						String, String, String, Text, String,
+//					),
-// 					),
+//					nil,
-// 					nil,
+//				},
-// 				},
+//			},
-// 			},
+//		},
-// 		},
+//	))
-// 	))
 //
-// See the lexers/m/markdown.go for the complete example.
+// See the lexers/markdown.go for the complete example.
 //
 // Note: panic's if the number of emitters does not equal the number of matched
 // groups in the regex.

backend/vendor/github.com/alecthomas/chroma/v2/formatters/html/html.go

@@ -5,7 +5,9 @@ import (
 	"html"
 	"io"
 	"sort"
+	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/alecthomas/chroma/v2"
 )
@@ -100,9 +102,9 @@ func LineNumbersInTable(b bool) Option {
 	}
 }
 
-// LinkableLineNumbers decorates the line numbers HTML elements with an "id"
+// WithLinkableLineNumbers decorates the line numbers HTML elements with an "id"
 // attribute so they can be linked.
-func LinkableLineNumbers(b bool, prefix string) Option {
+func WithLinkableLineNumbers(b bool, prefix string) Option {
 	return func(f *Formatter) {
 		f.linkableLineNumbers = b
 		f.lineNumbersIDPrefix = prefix
@@ -132,6 +134,7 @@ func New(options ...Option) *Formatter {
 		baseLineNumber: 1,
 		preWrapper:     defaultPreWrapper,
 	}
+	f.styleCache = newStyleCache(f)
 	for _, option := range options {
 		option(f)
 	}
@@ -171,10 +174,10 @@ var (
 	defaultPreWrapper = preWrapper{
 		start: func(code bool, styleAttr string) string {
 			if code {
-				return fmt.Sprintf(`<pre tabindex="0"%s><code>`, styleAttr)
+				return fmt.Sprintf(`<pre%s><code>`, styleAttr)
 			}
 
-			return fmt.Sprintf(`<pre tabindex="0"%s>`, styleAttr)
+			return fmt.Sprintf(`<pre%s>`, styleAttr)
 		},
 		end: func(code bool) string {
 			if code {
@@ -188,6 +191,7 @@ var (
 
 // Formatter that generates HTML.
 type Formatter struct {
+	styleCache            *styleCache
 	standalone            bool
 	prefix                string
 	Classes               bool // Exported field to detect when classes are being used
@@ -220,12 +224,7 @@ func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Ite
 //
 // OTOH we need to be super careful about correct escaping...
 func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.Token) (err error) { // nolint: gocyclo
-	css := f.styleToCSS(style)
+	css := f.styleCache.get(style, true)
-	if !f.Classes {
-		for t, style := range css {
-			css[t] = compressStyle(style)
-		}
-	}
 	if f.standalone {
 		fmt.Fprint(w, "<html>\n")
 		if f.Classes {
@@ -243,7 +242,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 	wrapInTable := f.lineNumbers && f.lineNumbersInTable
 
 	lines := chroma.SplitTokensIntoLines(tokens)
-	lineDigits := len(fmt.Sprintf("%d", f.baseLineNumber+len(lines)-1))
+	lineDigits := len(strconv.Itoa(f.baseLineNumber + len(lines) - 1))
 	highlightIndex := 0
 
 	if wrapInTable {
@@ -251,7 +250,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 		fmt.Fprintf(w, "<div%s>\n", f.styleAttr(css, chroma.PreWrapper))
 		fmt.Fprintf(w, "<table%s><tr>", f.styleAttr(css, chroma.LineTable))
 		fmt.Fprintf(w, "<td%s>\n", f.styleAttr(css, chroma.LineTableTD))
-		fmt.Fprintf(w, f.preWrapper.Start(false, f.styleAttr(css, chroma.PreWrapper)))
+		fmt.Fprintf(w, "%s", f.preWrapper.Start(false, f.styleAttr(css, chroma.PreWrapper)))
 		for index := range lines {
 			line := f.baseLineNumber + index
 			highlight, next := f.shouldHighlight(highlightIndex, line)
@@ -262,7 +261,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 				fmt.Fprintf(w, "<span%s>", f.styleAttr(css, chroma.LineHighlight))
 			}
 
-			fmt.Fprintf(w, "<span%s%s>%s\n</span>", f.styleAttr(css, chroma.LineNumbersTable), f.lineIDAttribute(line), f.lineTitleWithLinkIfNeeded(lineDigits, line))
+			fmt.Fprintf(w, "<span%s%s>%s\n</span>", f.styleAttr(css, chroma.LineNumbersTable), f.lineIDAttribute(line), f.lineTitleWithLinkIfNeeded(css, lineDigits, line))
 
 			if highlight {
 				fmt.Fprintf(w, "</span>")
@@ -273,7 +272,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 		fmt.Fprintf(w, "<td%s>\n", f.styleAttr(css, chroma.LineTableTD, "width:100%"))
 	}
 
-	fmt.Fprintf(w, f.preWrapper.Start(true, f.styleAttr(css, chroma.PreWrapper)))
+	fmt.Fprintf(w, "%s", f.preWrapper.Start(true, f.styleAttr(css, chroma.PreWrapper)))
 
 	highlightIndex = 0
 	for index, tokens := range lines {
@@ -302,7 +301,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 
 			// Line number
 			if f.lineNumbers && !wrapInTable {
-				fmt.Fprintf(w, "<span%s%s>%s</span>", f.styleAttr(css, chroma.LineNumbers), f.lineIDAttribute(line), f.lineTitleWithLinkIfNeeded(lineDigits, line))
+				fmt.Fprintf(w, "<span%s%s>%s</span>", f.styleAttr(css, chroma.LineNumbers), f.lineIDAttribute(line), f.lineTitleWithLinkIfNeeded(css, lineDigits, line))
 			}
 
 			fmt.Fprintf(w, `<span%s>`, f.styleAttr(css, chroma.CodeLine))
@@ -323,7 +322,7 @@ func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.
 			fmt.Fprint(w, `</span>`) // End of Line
 		}
 	}
-	fmt.Fprintf(w, f.preWrapper.End(true))
+	fmt.Fprintf(w, "%s", f.preWrapper.End(true))
 
 	if wrapInTable {
 		fmt.Fprint(w, "</td></tr></table>\n")
@@ -345,12 +344,12 @@ func (f *Formatter) lineIDAttribute(line int) string {
 	return fmt.Sprintf(" id=\"%s\"", f.lineID(line))
 }
 
-func (f *Formatter) lineTitleWithLinkIfNeeded(lineDigits, line int) string {
+func (f *Formatter) lineTitleWithLinkIfNeeded(css map[chroma.TokenType]string, lineDigits, line int) string {
 	title := fmt.Sprintf("%*d", lineDigits, line)
 	if !f.linkableLineNumbers {
 		return title
 	}
-	return fmt.Sprintf("<a style=\"outline: none; text-decoration:none; color:inherit\" href=\"#%s\">%s</a>", f.lineID(line), title)
+	return fmt.Sprintf("<a%s href=\"#%s\">%s</a>", f.styleAttr(css, chroma.LineLink), f.lineID(line), title)
 }
 
 func (f *Formatter) lineID(line int) string {
@@ -419,7 +418,7 @@ func (f *Formatter) tabWidthStyle() string {
 
 // WriteCSS writes CSS style definitions (without any surrounding HTML).
 func (f *Formatter) WriteCSS(w io.Writer, style *chroma.Style) error {
-	css := f.styleToCSS(style)
+	css := f.styleCache.get(style, false)
 	// Special-case background as it is mapped to the outer ".chroma" class.
 	if _, err := fmt.Fprintf(w, "/* %s */ .%sbg { %s }\n", chroma.Background, f.prefix, css[chroma.Background]); err != nil {
 		return err
@@ -513,13 +512,14 @@ func (f *Formatter) styleToCSS(style *chroma.Style) map[chroma.TokenType]string
 	if f.wrapLongLines {
 		classes[chroma.PreWrapper] += `white-space: pre-wrap; word-break: break-word;`
 	}
-	lineNumbersStyle := `white-space: pre; user-select: none; margin-right: 0.4em; padding: 0 0.4em 0 0.4em;`
+	lineNumbersStyle := `white-space: pre; -webkit-user-select: none; user-select: none; margin-right: 0.4em; padding: 0 0.4em 0 0.4em;`
 	// All rules begin with default rules followed by user provided rules
 	classes[chroma.Line] = `display: flex;` + classes[chroma.Line]
 	classes[chroma.LineNumbers] = lineNumbersStyle + classes[chroma.LineNumbers]
 	classes[chroma.LineNumbersTable] = lineNumbersStyle + classes[chroma.LineNumbersTable]
 	classes[chroma.LineTable] = "border-spacing: 0; padding: 0; margin: 0; border: 0;" + classes[chroma.LineTable]
 	classes[chroma.LineTableTD] = "vertical-align: top; padding: 0; margin: 0; border: 0;" + classes[chroma.LineTableTD]
+	classes[chroma.LineLink] = "outline: none; text-decoration: none; color: inherit" + classes[chroma.LineLink]
 	return classes
 }
 
@@ -561,3 +561,63 @@ func compressStyle(s string) string {
 	}
 	return strings.Join(out, ";")
 }
+
+const styleCacheLimit = 32
+
+type styleCacheEntry struct {
+	style      *chroma.Style
+	compressed bool
+	cache      map[chroma.TokenType]string
+}
+
+type styleCache struct {
+	mu sync.Mutex
+	// LRU cache of compiled (and possibly compressed) styles. This is a slice
+	// because the cache size is small, and a slice is sufficiently fast for
+	// small N.
+	cache []styleCacheEntry
+	f     *Formatter
+}
+
+func newStyleCache(f *Formatter) *styleCache {
+	return &styleCache{f: f}
+}
+
+func (l *styleCache) get(style *chroma.Style, compress bool) map[chroma.TokenType]string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	// Look for an existing entry.
+	for i := len(l.cache) - 1; i >= 0; i-- {
+		entry := l.cache[i]
+		if entry.style == style && entry.compressed == compress {
+			// Top of the cache, no need to adjust the order.
+			if i == len(l.cache)-1 {
+				return entry.cache
+			}
+			// Move this entry to the end of the LRU
+			copy(l.cache[i:], l.cache[i+1:])
+			l.cache[len(l.cache)-1] = entry
+			return entry.cache
+		}
+	}
+
+	// No entry, create one.
+	cached := l.f.styleToCSS(style)
+	if !l.f.Classes {
+		for t, style := range cached {
+			cached[t] = compressStyle(style)
+		}
+	}
+	if compress {
+		for t, style := range cached {
+			cached[t] = compressStyle(style)
+		}
+	}
+	// Evict the oldest entry.
+	if len(l.cache) >= styleCacheLimit {
+		l.cache = l.cache[0:copy(l.cache, l.cache[1:])]
+	}
+	l.cache = append(l.cache, styleCacheEntry{style: style, cache: cached, compressed: compress})
+	return cached
+}

backend/vendor/github.com/alecthomas/chroma/v2/lexer.go

@@ -57,6 +57,26 @@ type Config struct {
 	//
 	// If this is 0 it will be treated as a default of 1.
 	Priority float32 `xml:"priority,omitempty"`
+
+	// Analyse is a list of regexes to match against the input.
+	//
+	// If a match is found, the score is returned if single attribute is set to true,
+	// otherwise the sum of all the score of matching patterns will be
+	// used as the final score.
+	Analyse *AnalyseConfig `xml:"analyse,omitempty"`
+}
+
+// AnalyseConfig defines the list of regexes analysers.
+type AnalyseConfig struct {
+	Regexes []RegexConfig `xml:"regex,omitempty"`
+	// If true, the first matching score is returned.
+	First bool `xml:"first,attr"`
+}
+
+// RegexConfig defines a single regex pattern and its score in case of match.
+type RegexConfig struct {
+	Pattern string  `xml:"pattern,attr"`
+	Score   float32 `xml:"score,attr"`
 }
 
 // Token output to formatter.

backend/vendor/github.com/alecthomas/chroma/v2/lexers/README.md

@@ -1,19 +1,24 @@
-# Lexer tests
+# Chroma lexers
+
+All lexers in Chroma should now be defined in XML unless they require custom code.
+
+## Lexer tests
 
 The tests in this directory feed a known input `testdata/<name>.actual` into the parser for `<name>` and check
-that its output matches `<name>.exported`.
+that its output matches `<name>.expected`.
 
 It is also possible to perform several tests on a same parser `<name>`, by placing know inputs `*.actual` into a
 directory `testdata/<name>/`.
 
-## Running the tests
+### Running the tests
 
 Run the tests as normal:
 ```go
 go test ./lexers
 ```
 
-## Update existing tests
+### Update existing tests
+
 When you add a new test data file (`*.actual`), you need to regenerate all tests. That's how Chroma creates the `*.expected` test file based on the corresponding lexer.
 
 To regenerate all tests, type in your terminal:
@@ -26,7 +31,8 @@ This first sets the `RECORD` environment variable to `true`. Then it runs `go te
 
 (That environment variable tells Chroma it needs to output test data. After running `go test ./lexers` you can remove or reset that variable.)
 
-### Windows users
+#### Windows users
+
 Windows users will find that the `RECORD=true go test ./lexers` command fails in both the standard command prompt terminal and in PowerShell.
 
 Instead we have to perform both steps separately:
@@ -35,6 +41,6 @@ Instead we have to perform both steps separately:
 	+ In the regular command prompt window, the `set` command sets an environment variable for the current session: `set RECORD=true`. See [this page](https://superuser.com/questions/212150/how-to-set-env-variable-in-windows-cmd-line) for more.
 	+ In PowerShell, you can use the `$env:RECORD = 'true'` command for that. See [this article](https://mcpmag.com/articles/2019/03/28/environment-variables-in-powershell.aspx) for more.
 	+ You can also make a persistent environment variable by hand in the Windows computer settings. See [this article](https://www.computerhope.com/issues/ch000549.htm) for how.
-- When the environment variable is set, run `go tests ./lexers`.
+- When the environment variable is set, run `go test ./lexers`.
 
 Chroma will now regenerate the test files and print its results to the console window.