Compare commits
73
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3286f028e3 | ||
|
|
c947a06d48 | ||
|
|
5f439cf1d5 | ||
|
|
7de51ad2ef | ||
|
|
389189afcf | ||
|
|
6f40926241 | ||
|
|
2640aaee9e | ||
|
|
250e8ad925 | ||
|
|
4793c1cd2c | ||
|
|
23be25d84a | ||
|
|
4d5dae20e2 | ||
|
|
a82dc8d3f1 | ||
|
|
e3d1d0e23e | ||
|
|
49bf83b09c | ||
|
|
0b651dddd4 | ||
|
|
b6975af63c | ||
|
|
4fca268dc6 | ||
|
|
a0879b2336 | ||
|
|
7c312da199 | ||
|
|
a5b2fe3cfc | ||
|
|
7ee3a0da48 | ||
|
|
4aaba0fb61 | ||
|
|
fb7b6c2cdd | ||
|
|
123b9328d9 | ||
|
|
2bfad021e3 | ||
|
|
4c9ef37cf1 | ||
|
|
ff77f41a3a | ||
|
|
1bb002348a | ||
|
|
fc4e10573c | ||
|
|
a91e322d5c | ||
|
|
931f2db4e3 | ||
|
|
b8f6dc5f91 | ||
|
|
b03dc366f9 | ||
|
|
36062de0e7 | ||
|
|
90766d6637 | ||
|
|
a1dbb2cb92 | ||
|
|
d370b78613 | ||
|
|
439ccfa83c | ||
|
|
29627f4bf0 | ||
|
|
164eb89c60 | ||
|
|
09110c792f | ||
|
|
3f5b3cdd98 | ||
|
|
43fccf3bc9 | ||
|
|
1f34984dab | ||
|
|
455d770899 | ||
|
|
bf67c251c5 | ||
|
|
cebba4cee4 | ||
|
|
35a389cb75 | ||
|
|
a725d990ed | ||
|
|
fdfce6495c | ||
|
|
8801903d01 | ||
|
|
8bcfd9e456 | ||
|
|
5b37a583ce | ||
|
|
29b5f88a1c | ||
|
|
287aef4dfb | ||
|
|
d06aa6771c | ||
|
|
a54d2d2756 | ||
|
|
e575066ea9 | ||
|
|
2544d80f98 | ||
|
|
5c0798fe10 | ||
|
|
76d0cc2cf6 | ||
|
|
51b6a7e890 | ||
|
|
8c5e82bd16 | ||
|
|
95f59213e5 | ||
|
|
503f5cacb0 | ||
|
|
d1f8cf412b | ||
|
|
db9d8703ef | ||
|
|
8f61ec691b | ||
|
|
07f6b9a0a0 | ||
|
|
98e4f03091 | ||
|
|
f8f2becb4b | ||
|
|
6e7820d2b7 | ||
|
|
3e63d72852 |
+3
-5
@@ -10,10 +10,6 @@
|
||||
/frontend/node_modules/
|
||||
/frontend/apps/remark42/node_modules/
|
||||
/frontend/apps/remark42/public/
|
||||
# e2e tests arficats
|
||||
/frontend/e2e/playwright-report/
|
||||
/frontend/e2e/playwright/.cache/
|
||||
/frontend/e2e/test-results/
|
||||
|
||||
# source files
|
||||
docker-compose.yml
|
||||
@@ -36,4 +32,6 @@ debug.test
|
||||
*.test
|
||||
remark42
|
||||
/backend/var/
|
||||
/playwright-report/
|
||||
|
||||
# go e2e suite, never built into the image
|
||||
/e2e/
|
||||
|
||||
+1
-1
@@ -3,4 +3,4 @@
|
||||
# review when someone opens a pull request.
|
||||
|
||||
* @umputun
|
||||
frontend/* @akellbl4 @Mavrin
|
||||
frontend/* @umputun @akellbl4 @Mavrin
|
||||
|
||||
+20
-26
@@ -4,6 +4,11 @@
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
|
||||
# npm updates are switched off entirely. open-pull-requests-limit bounds version
|
||||
# updates only, so the ignore entries below are what also stops security updates;
|
||||
# removing the npm entries would not work, as security updates come from alerts
|
||||
# rather than from this file.
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
@@ -20,29 +25,18 @@ updates:
|
||||
groups:
|
||||
"Go modules updates":
|
||||
dependency-type: "production"
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/e2e"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"Go modules updates":
|
||||
dependency-type: "production"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend"
|
||||
open-pull-requests-limit: 0
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"NPM modules updates":
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/packages/api"
|
||||
open-pull-requests-limit: 0
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"NPM modules updates":
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/e2e"
|
||||
open-pull-requests-limit: 0
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
@@ -53,6 +47,8 @@ updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/apps/remark42"
|
||||
open-pull-requests-limit: 0
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
@@ -60,13 +56,11 @@ updates:
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
- package-ecosystem: "npm"
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/site"
|
||||
open-pull-requests-limit: 0
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"NPM modules updates":
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
"Site image updates":
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
@@ -7,13 +7,18 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/ci-backend.yml"
|
||||
- "backend/**"
|
||||
- "Dockerfile"
|
||||
- "docker-init.sh"
|
||||
- ".dockerignore"
|
||||
- "!backend/scripts/**"
|
||||
- "!**.md"
|
||||
pull_request:
|
||||
types: [opened, reopened]
|
||||
paths:
|
||||
- ".github/workflows/ci-backend.yml"
|
||||
- "backend/**"
|
||||
- "Dockerfile"
|
||||
- "docker-init.sh"
|
||||
- ".dockerignore"
|
||||
- "!backend/scripts/**"
|
||||
- "!**.md"
|
||||
|
||||
@@ -35,14 +40,15 @@ jobs:
|
||||
DEBUG: ${{secrets.DEBUG}}
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend
|
||||
|
||||
- name: test and build backend
|
||||
run: |
|
||||
go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
|
||||
go test -race -timeout=300s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
|
||||
cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
|
||||
go build -race ./...
|
||||
working-directory: backend/app
|
||||
@@ -60,13 +66,13 @@ jobs:
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: "v2.10.1"
|
||||
version: "v2.13.1"
|
||||
working-directory: backend/app
|
||||
|
||||
- name: golangci-lint on example directory
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: "v2.10.1"
|
||||
version: "v2.13.1"
|
||||
args: --config ../../.golangci.yml
|
||||
working-directory: backend/_example/memory_store
|
||||
|
||||
@@ -77,3 +83,35 @@ jobs:
|
||||
working-directory: backend
|
||||
env:
|
||||
COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
vulncheck:
|
||||
name: Vulnerability scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
# both go.sum files so the cache key covers the main and example modules scanned below
|
||||
cache-dependency-path: |
|
||||
backend/go.sum
|
||||
backend/_example/memory_store/go.sum
|
||||
|
||||
- name: govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.5.0
|
||||
govulncheck ./...
|
||||
(cd _example/memory_store && govulncheck ./...)
|
||||
working-directory: backend
|
||||
env:
|
||||
# ignore the committed vendor dirs and resolve modules from the cache so
|
||||
# both the main module and the nested example module scan consistently
|
||||
GOFLAGS: "-mod=readonly"
|
||||
|
||||
@@ -30,14 +30,6 @@ jobs:
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: expose GitHub Actions cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-
|
||||
|
||||
- name: free disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
@@ -46,21 +38,26 @@ jobs:
|
||||
docker system prune -af
|
||||
|
||||
- name: build docker image without pushing
|
||||
run: |
|
||||
docker buildx build --load \
|
||||
--cache-from type=local,src=/tmp/.buildx-cache \
|
||||
--cache-to type=local,dest=/tmp/.buildx-cache-new,mode=max \
|
||||
--build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
|
||||
--platform linux/amd64 .
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
cache-from: type=gha,scope=main
|
||||
cache-to: type=gha,scope=main,mode=max,ignore-error=true
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
|
||||
- name: build example docker image without pushing
|
||||
run: |
|
||||
docker buildx build --load \
|
||||
--cache-from type=local,src=/tmp/.buildx-cache \
|
||||
--build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true \
|
||||
--platform linux/amd64 -f backend/_example/memory_store/Dockerfile .
|
||||
|
||||
- name: rotate cache
|
||||
run: |
|
||||
rm -rf /tmp/.buildx-cache
|
||||
mv /tmp/.buildx-cache-new /tmp/.buildx-cache || true
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: backend/_example/memory_store/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
cache-from: type=gha,scope=example
|
||||
cache-to: type=gha,scope=example,mode=max,ignore-error=true
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
name: compose
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-compose.yml"
|
||||
- "**compose*.yml"
|
||||
- "**compose*.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-compose.yml"
|
||||
- "**compose*.yml"
|
||||
- "**compose*.yaml"
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate compose files
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: validate tracked compose files
|
||||
run: |
|
||||
set -euo pipefail
|
||||
n=0
|
||||
# null-delimited to stay safe with unusual filenames; exclude this
|
||||
# workflow (its name contains "compose") and vendored compose files.
|
||||
# filenames are not echoed as workflow commands to avoid log-command injection
|
||||
while IFS= read -r -d '' f; do
|
||||
docker compose -f "$f" config --quiet
|
||||
n=$((n + 1))
|
||||
done < <(git ls-files -z '*compose*.yml' '*compose*.yaml' ':!:*/vendor/*' ':!:.github/*')
|
||||
if [ "$n" -eq 0 ]; then
|
||||
echo "no compose files found" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "validated $n compose file(s)"
|
||||
@@ -0,0 +1,37 @@
|
||||
name: docs versions
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-docs-versions.yml"
|
||||
- "scripts/check-documented-versions.sh"
|
||||
- "site/content/docs/getting-started/installation/index.md"
|
||||
- "backend/go.mod"
|
||||
- "frontend/apps/remark42/package.json"
|
||||
- "frontend/.nvmrc"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-docs-versions.yml"
|
||||
- "scripts/check-documented-versions.sh"
|
||||
- "site/content/docs/getting-started/installation/index.md"
|
||||
- "backend/go.mod"
|
||||
- "frontend/apps/remark42/package.json"
|
||||
- "frontend/.nvmrc"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Documented versions
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check documented versions against the repository
|
||||
run: ./scripts/check-documented-versions.sh
|
||||
@@ -1,131 +0,0 @@
|
||||
name: "@remark42/api"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend-api.yml"
|
||||
- "frontend/packages/**"
|
||||
- "!**.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend-api.yml"
|
||||
- "frontend/packages/**"
|
||||
- "!**.md"
|
||||
|
||||
jobs:
|
||||
type-check:
|
||||
name: Type check
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [ 20 ]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
|
||||
- name: Run type check
|
||||
run: pnpm type-check:api
|
||||
working-directory: ./frontend
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [ 20 ]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
|
||||
- name: Run linters
|
||||
run: pnpm lint:api
|
||||
working-directory: ./frontend/
|
||||
|
||||
test:
|
||||
name: Tests & Coverage
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [ 20 ]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
|
||||
- name: Test & Coverage
|
||||
run: pnpm coverage:api
|
||||
working-directory: ./frontend
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v7
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
working-directory: ./frontend
|
||||
codecov_yml_path: ./frontend/apps/remark42/codecov.yml
|
||||
@@ -6,12 +6,12 @@ on:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend.yml"
|
||||
- "frontend/apps/remark42/**"
|
||||
- "frontend/**"
|
||||
- "!**.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend.yml"
|
||||
- "frontend/apps/remark42/**"
|
||||
- "frontend/**"
|
||||
- "!**.md"
|
||||
|
||||
jobs:
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [20]
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -31,21 +31,21 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Translations check
|
||||
run: pnpm translation-check
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [20]
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -67,21 +67,21 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Run type check
|
||||
run: pnpm type-check
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [20]
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -103,21 +103,21 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Run linters
|
||||
run: pnpm lint
|
||||
@@ -139,7 +139,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
@@ -158,7 +158,7 @@ jobs:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [20]
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -167,21 +167,21 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm i
|
||||
working-directory: ./frontend
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Test & Coverage
|
||||
run: pnpm coverage
|
||||
|
||||
@@ -9,16 +9,47 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/ci-site.yml"
|
||||
- "site/**"
|
||||
- "!**/CLAUDE.md"
|
||||
- "!site/README.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-site.yml"
|
||||
- "site/**"
|
||||
- "!**/CLAUDE.md"
|
||||
- "!site/README.md"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Build site image (pull request)
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: set up docker buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: build image without pushing
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./site
|
||||
load: true
|
||||
push: false
|
||||
cache-from: |
|
||||
type=gha,scope=site-pr
|
||||
type=gha,scope=site-linux/amd64
|
||||
cache-to: type=gha,scope=site-pr,mode=max,ignore-error=true
|
||||
|
||||
build:
|
||||
name: Build site image (${{ matrix.platform }})
|
||||
if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
|
||||
@@ -60,7 +91,7 @@ jobs:
|
||||
context: ./site
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=site-${{ matrix.platform }}
|
||||
cache-to: type=gha,scope=site-${{ matrix.platform }},mode=max
|
||||
cache-to: type=gha,scope=site-${{ matrix.platform }},mode=max,ignore-error=true
|
||||
outputs: type=image,name=ghcr.io/umputun/remark42-site,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: export digest
|
||||
|
||||
@@ -2,7 +2,7 @@ name: docker
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [backend]
|
||||
workflows: [backend, frontend]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
|
||||
+100
-17
@@ -5,22 +5,34 @@ on:
|
||||
branches: [master]
|
||||
paths:
|
||||
- ".github/workflows/e2e-tests.yml"
|
||||
- "frontend/apps/remark42/**"
|
||||
- "frontend/e2e/**"
|
||||
- "frontend/Dockerfile.e2e"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "e2e/**"
|
||||
- "compose-e2e-test.yml"
|
||||
- "Dockerfile"
|
||||
- "!**.md"
|
||||
|
||||
pull_request:
|
||||
branches: [master]
|
||||
paths:
|
||||
- ".github/workflows/e2e-tests.yml"
|
||||
- "frontend/apps/remark42/**"
|
||||
- "frontend/e2e/**"
|
||||
- "frontend/Dockerfile.e2e"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "e2e/**"
|
||||
- "compose-e2e-test.yml"
|
||||
- "Dockerfile"
|
||||
- "!**.md"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Tests
|
||||
timeout-minutes: 60
|
||||
# cheap gate: catches a compile break or a lint regression in the build-tagged suite
|
||||
# without paying for the docker build and the browser download
|
||||
vet:
|
||||
name: Vet
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,13 +42,84 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build & run containers
|
||||
id: tests
|
||||
run: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
name: playwright-report
|
||||
path: ./playwright-report/
|
||||
go-version-file: e2e/go.mod
|
||||
cache-dependency-path: e2e/go.sum
|
||||
|
||||
- name: Vet
|
||||
run: cd e2e && go vet -tags=e2e ./...
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.13.1
|
||||
working-directory: e2e
|
||||
args: --build-tags=e2e --config ../backend/.golangci.yml
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
needs: vet
|
||||
# generous against the docker build plus one 8m go test: a job cancelled on timeout skips
|
||||
# its own failure steps, so the run would end with neither logs nor traces
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: e2e/go.mod
|
||||
cache-dependency-path: e2e/go.sum
|
||||
|
||||
# two directories: the driver (node plus the npm package) and the browser builds,
|
||||
# which include firefox and webkit for the rendering tests
|
||||
- name: Cache playwright driver and browsers
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
~/.cache/ms-playwright-go
|
||||
key: playwright-${{ hashFiles('e2e/go.sum') }}
|
||||
restore-keys: playwright-
|
||||
|
||||
# E2E_STAMP is what the suite compares the running stack against, so a stack started here
|
||||
# has to carry the same value `make e2e-up` and the suite itself would give it
|
||||
- name: Build & start the stack
|
||||
run: |
|
||||
./e2e/tls/generate.sh
|
||||
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 E2E_STAMP=$(./e2e/stamp.sh) \
|
||||
docker compose -f compose-e2e-test.yml up -d --build --quiet-pull --wait
|
||||
|
||||
# no retry: a failure here is evidence about a suite too young to have a flake rate,
|
||||
# and a rerun is how an intermittent regression becomes invisible. revisit when there
|
||||
# are failures on record to look at
|
||||
- name: Run e2e
|
||||
# stamps this run's comment threads with the CI run, so a thread url in a trace or a
|
||||
# log names the run it came from
|
||||
env:
|
||||
E2E_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# 20m, matching the Makefile. the suite runs about four minutes on a laptop and a runner
|
||||
# is slower, so a tighter budget turns a loaded runner into a timeout panic instead of a
|
||||
# readable failure. the job's own timeout above is what bounds a wedged run
|
||||
run: cd e2e && go test -tags=e2e -count 1 -timeout 20m -v ./...
|
||||
|
||||
- name: Server logs on failure
|
||||
if: failure()
|
||||
run: docker compose -f compose-e2e-test.yml logs --tail=200
|
||||
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: playwright-traces
|
||||
path: e2e/traces/
|
||||
retention-days: 30
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -12,10 +12,10 @@ on:
|
||||
- "scripts/**"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "!backend/**.md"
|
||||
- "!frontend/**.md"
|
||||
- "README.md"
|
||||
- "LICENSE"
|
||||
- "CLAUDE.md"
|
||||
- "site/src/docs/getting-started/installation/index.md"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,27 +30,28 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 24
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: test and build backend
|
||||
run: |
|
||||
go test -race -timeout=120s ./...
|
||||
go test -race -timeout=300s ./...
|
||||
go build -race ./...
|
||||
working-directory: backend/app
|
||||
env:
|
||||
@@ -66,7 +67,7 @@ jobs:
|
||||
|
||||
- name: install frontend dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: frontend
|
||||
working-directory: frontend/apps/remark42
|
||||
env:
|
||||
CI: "true"
|
||||
|
||||
@@ -105,27 +106,28 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: install pnpm
|
||||
uses: pnpm/action-setup@v6.0.9
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: install node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 24
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: install frontend dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: frontend
|
||||
working-directory: frontend/apps/remark42
|
||||
env:
|
||||
CI: "true"
|
||||
|
||||
|
||||
+6
-1
@@ -25,8 +25,13 @@ compose-private-frontend.yml
|
||||
compose-private.yml
|
||||
/backend/_example/*/vendor
|
||||
http-client.env.json
|
||||
/playwright-report/
|
||||
/backend/app/cmd/var
|
||||
|
||||
# ralphex progress logs
|
||||
.ralphex/progress/
|
||||
|
||||
# traces from failed e2e runs
|
||||
/e2e/traces/
|
||||
|
||||
# self-signed certificate for the e2e https services, made by e2e/tls/generate.sh
|
||||
/e2e/tls/*.pem
|
||||
|
||||
@@ -26,26 +26,17 @@ builds:
|
||||
- amd64
|
||||
- arm64
|
||||
- "386"
|
||||
- arm
|
||||
goarm:
|
||||
- "7"
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: "386"
|
||||
- goos: darwin
|
||||
goarch: arm
|
||||
- goos: freebsd
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: "386"
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: "386"
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
ldflags:
|
||||
- -s -w -X main.revision={{ .Tag }}-{{ .ShortCommit }}-{{ trimsuffix (replace (replace .CommitDate "-" "") ":" "") "Z" }}
|
||||
|
||||
|
||||
@@ -6,19 +6,40 @@
|
||||
- Build: `make backend`
|
||||
- Race test: `make race_test`
|
||||
- **Backend Testing**:
|
||||
- Run all tests: `cd backend/app && go test -timeout=60s -count 1 ./...`
|
||||
- Run all tests: `cd backend/app && go test -timeout=300s -count 1 ./...`
|
||||
- Run single test: `cd backend/app && go test -run TestName ./path/to/package`
|
||||
- **IMPORTANT**: Run example tests: `cd backend/_example/memory_store && go test -race ./... && go build -race ./...`
|
||||
- **Frontend**:
|
||||
- Development: `cd frontend && pnpm dev:app`
|
||||
- Tests: `cd frontend && pnpm test`
|
||||
- Development: `cd frontend/apps/remark42 && pnpm dev`
|
||||
- Tests: `cd frontend/apps/remark42 && pnpm test`
|
||||
- **End-to-end**: `make e2e` drives the widget in a real browser; see `e2e/README.md`. Build-tagged, so `go test ./...` never runs it.
|
||||
- **Lint**:
|
||||
- Backend: `cd backend && golangci-lint run`
|
||||
- **IMPORTANT**: Example lint: `cd backend/_example/memory_store && golangci-lint run --config ../../.golangci.yml`
|
||||
- Frontend: `cd frontend && pnpm lint`
|
||||
- Frontend: `cd frontend/apps/remark42 && pnpm lint`
|
||||
- **Before committing**: Always run tests and linter on both main backend AND examples
|
||||
- **Dependency Updates**:
|
||||
- When updating Go modules in `backend/`, also run `go mod tidy` (and `go mod vendor`) in `backend/_example/memory_store` to keep indirect deps in sync. The example module replaces `github.com/umputun/remark42/backend` with `../../` so stale indirect deps there will break the example build.
|
||||
- **Go module changes**:
|
||||
- **Any** change to `backend/go.mod` or `backend/go.sum` requires `go mod tidy` in `backend/_example/memory_store` in the same commit. That covers dependency bumps, adding or removing a dependency, and changing the `go` directive, not only version updates.
|
||||
- Only `go mod tidy` there, not `go mod vendor`: the example's vendor directory is gitignored (`.gitignore:26`), so its output is never committed, while a stale local copy silently becomes what the example resolves against.
|
||||
- The example module replaces `github.com/umputun/remark42/backend` with `../../`, so it carries the backend's dependencies as indirect entries. Leaving them stale fails the `test examples` CI step with `go: updates to go.mod needed; to update it: go mod tidy`.
|
||||
- This applies to Dependabot pull requests too: the bot updates `backend/` only, so its Go module PRs need the example tidied before they can go green.
|
||||
|
||||
|
||||
## Backend Test Determinism
|
||||
|
||||
Backend tests must never depend on how fast the machine is. CI runs them under `-race` with coverage on a shared runner, so any test that assumes an operation finishes within some duration eventually fails on a rerun-and-it-passes basis.
|
||||
|
||||
- **Wait on a condition, never on a duration.** Use `require.Eventually` / `require.EventuallyWithT` to poll for the state the assertion needs, and `require.Never` when the point is that something did *not* happen. A bare `time.Sleep` before an assertion is a defect; sleeping until a deadline you computed, as `waitPastMillisecond` does, is not.
|
||||
- **Polling closures must not touch `*testing.T`.** testify runs them on a separate goroutine, where `t.FailNow` is undefined behaviour. Assert on the `*assert.CollectT` that `EventuallyWithT` hands the closure, so the real error also lands in the failure message.
|
||||
- **Mind the rate limiter when polling over HTTP.** Route groups are capped independently and most of the caps are hard-coded in `rest.go`, out of reach of a test: `/auth/` at 2 req/s and the admin, protected and image routes at 10 req/s. Only the open-route group is settable, via `openRouteLimiter` (100 in `startupT`). Poll with the existing constants rather than a new number, `httpPoll` for anything issuing an HTTP request and `pollInterval` only for in-process or filesystem checks, or the poll manufactures the 429s it then has to interpret.
|
||||
- **When a test needs time to have passed, pin the clock input rather than waiting for it:** `os.Chtimes` for file ages, an explicit `store.Comment.Timestamp` for anything that formats a timestamp.
|
||||
- **Prefer a `testing/synctest` bubble** where the code under test has no real I/O. Inside one the clock is fake, so `time.Sleep` is instant and deterministic. `app/notify`, `app/store/service`, `app/store/image`, `app/store/engine`, `app/providers`, `app/migrator` and `_example/memory_store/accessor` already use it, and most surviving `time.Sleep` calls live in them.
|
||||
- **Helpers fail loudly.** A wait that gives up must call `t.Fatal`/`require` naming what it was waiting for, never return silently and leave the next assertion to fail with something unrelated. Because these packages run `goleak.VerifyTestMain`, a failing helper also exits the test goroutine, so anything that started a server in a goroutine must `defer cancel()` or `defer srv.Shutdown()` right after launching it; otherwise a failed readiness wait is reported as a goroutine leak rather than the failure that caused it.
|
||||
- **Take ports and paths from outside the test.** Ports come from the kernel with `net.Listen("tcp", ":0")`, files from `t.TempDir()`. `go test ./...` runs package binaries concurrently, so a number out of a fixed range or a fixed name under `/tmp` lets two of them collide.
|
||||
- **Close idle connections before shutting a test server down.** Clients built as `http.Client{Timeout: x}` share `http.DefaultTransport`, and `Shutdown` waits on their keep-alive connections until its own deadline expires.
|
||||
- **Keep the test timeout budgets aligned.** `Makefile`, `ci-backend.yml`, `release.yml` and the command above all use `-timeout=300s`; the wait helpers allow 30s per condition, so a shorter per-package budget turns a slow runner into a timeout panic instead of a readable failure.
|
||||
|
||||
`chooseUnusedPort` and the server-start wait helpers are duplicated in `app`, `app/cmd`, `app/rest/api` and `_example/memory_store/server`. Nothing shares them today; keep the copies in step when changing one.
|
||||
|
||||
## Release Procedure
|
||||
|
||||
@@ -39,7 +60,22 @@ git push origin backend/vX.Y.Z
|
||||
|
||||
GoReleaser must ignore `backend/*` tags in `.goreleaser.yml` so release notes and current-tag detection use only product tags. Docker image publishing stays separate and is handled by the existing Docker workflow.
|
||||
|
||||
For local artifact runs, install GoReleaser, Go 1.25, Node 16+, PNPM 8, and Perl, then use `make release`. The target runs a snapshot/no-publish GoReleaser build, leaves local artifacts and metadata in `dist/`, and cleans generated frontend embed files after GoReleaser exits. Do not run raw `goreleaser release` for local artifacts unless you also run `./scripts/cleanup-release-assets.sh` afterward.
|
||||
For local artifact runs, install GoReleaser, Go 1.25, Node 24+ and PNPM 10, then use `make release`. The target runs a snapshot/no-publish GoReleaser build, leaves local artifacts and metadata in `dist/`, and cleans generated frontend embed files after GoReleaser exits. Do not run raw `goreleaser release` for local artifacts unless you also run `./scripts/cleanup-release-assets.sh` afterward.
|
||||
|
||||
## Milestones and Issue Labels
|
||||
|
||||
**Milestones** — one `vX.Y.Z` milestone per release. Assign every merged PR, and every issue closed by a code change, to the milestone of the release it shipped in.
|
||||
- Decide which release a PR belongs to by whether its merge commit is **contained in a release tag** — not by comparing dates (a tag can be cut from an earlier commit, or moved). `git fetch --tags`, then `git tag --contains <merge_sha> | grep '^v' | sort -V | head -1` is its release. If no release tag contains it yet, it belongs to the next (unreleased) version's milestone — create it if missing (`gh api repos/umputun/remark42/milestones -f title="vX.Y.Z"`).
|
||||
- An **issue gets a milestone only when it was closed by a code change** (a linked closing PR/commit); take the milestone from that PR/commit (via the commit-in-tag rule). Issues closed as `duplicate`/`invalid`/`wontfix`/answered get no milestone.
|
||||
- Find unassigned: `gh pr list --state merged --search "no:milestone"`, `gh issue list --state closed --search "no:milestone"`. Assign with `gh pr edit N --milestone "vX.Y.Z"` / `gh issue edit N --milestone "vX.Y.Z"`.
|
||||
|
||||
**Issue labels** — classify each issue with a type and an area (add priority when relevant):
|
||||
- Type: `bug`, `enhancement`, `question`, `documentation`, `discussion`
|
||||
- Area: `backend`, `frontend`, `site`, `CI`, `design`, `localization`
|
||||
- Priority: `important`, `minor`, `some day`
|
||||
- Contribution: `help wanted`, `good-first-issue`
|
||||
- Resolution (on close, when applicable): `duplicate`, `invalid`, `wontfix`, `no-action-needed`
|
||||
- PR auto-labels (applied by Dependabot/Actions, not manual PRs): `dependencies`, `go`, `javascript`, `github_actions`
|
||||
|
||||
## Code Style
|
||||
- **Backend**: Formatting with golangci-lint, strict error handling
|
||||
@@ -57,3 +93,8 @@ For local artifact runs, install GoReleaser, Go 1.25, Node 16+, PNPM 8, and Perl
|
||||
## Repository Structure
|
||||
- Backend: Go server using BoltDB for storage
|
||||
- Frontend: Preact/Redux-based UI with iframe embedding
|
||||
- `/web` is served from two sources, in lookup order: the frontend build output
|
||||
(`frontend/apps/remark42/public`, embedded at `backend/app/cmd/web` or read from `--web-root`),
|
||||
then `backend/app/webassets/assets`, embedded in the binary. A plain page or image the bundler
|
||||
does not process belongs in `webassets`; anything needing templating or the widget's CSS/JS goes
|
||||
through webpack. A name present in both is served from the frontend build.
|
||||
|
||||
+6
-5
@@ -1,12 +1,14 @@
|
||||
FROM --platform=$BUILDPLATFORM node:20-alpine AS frontend-deps
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine AS frontend-deps
|
||||
|
||||
ARG SKIP_FRONTEND_TEST
|
||||
ARG SKIP_FRONTEND_BUILD
|
||||
# the manifest's prepare script installs husky hooks, which needs a git repository the build
|
||||
# context does not have. husky itself skips on CI, and this is the same flag the build stage sets
|
||||
ENV CI=true
|
||||
|
||||
WORKDIR /srv/frontend/
|
||||
WORKDIR /srv/frontend/apps/remark42/
|
||||
|
||||
COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml /srv/frontend/
|
||||
COPY ./frontend/apps/remark42/package.json /srv/frontend/apps/remark42/
|
||||
COPY ./frontend/apps/remark42/package.json ./frontend/apps/remark42/pnpm-lock.yaml /srv/frontend/apps/remark42/
|
||||
|
||||
RUN \
|
||||
if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
|
||||
@@ -60,7 +62,6 @@ RUN apk --no-cache add gcc libc-dev
|
||||
ADD backend /build/backend
|
||||
# to embed the frontend files statically into Remark42 binary
|
||||
COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
|
||||
RUN find /build/backend/app/cmd/web/ -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;
|
||||
WORKDIR /build/backend
|
||||
|
||||
RUN echo go version: `go version`
|
||||
|
||||
@@ -17,7 +17,7 @@ docker:
|
||||
dockerx:
|
||||
docker buildx build --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) --build-arg CI=true \
|
||||
--build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
|
||||
--progress=plain --platform linux/amd64,linux/arm/v7,linux/arm64 \
|
||||
--progress=plain --platform linux/amd64,linux/arm64 \
|
||||
-t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
|
||||
|
||||
release:
|
||||
@@ -26,7 +26,7 @@ release:
|
||||
goreleaser release --snapshot --clean --skip=publish
|
||||
|
||||
race_test:
|
||||
cd backend/app && go test -race -timeout=60s -count 1 ./...
|
||||
cd backend/app && go test -race -timeout=300s -count 1 ./...
|
||||
|
||||
backend:
|
||||
docker compose -f compose-dev-backend.yml build
|
||||
@@ -39,7 +39,21 @@ rundev:
|
||||
docker compose -f compose-private.yml build
|
||||
docker compose -f compose-private.yml up
|
||||
|
||||
e2e:
|
||||
docker compose -f compose-e2e-test.yml up --build --quiet-pull --exit-code-from tests
|
||||
# stamped the same way the suite stamps a stack it starts itself, so one brought up here is
|
||||
# accepted instead of rejected as belonging to another checkout
|
||||
e2e-up:
|
||||
./e2e/tls/generate.sh
|
||||
E2E_STAMP=$$(./e2e/stamp.sh) docker compose -f compose-e2e-test.yml up -d --build --quiet-pull --wait
|
||||
|
||||
.PHONY: bin docker dockerx release race_test backend frontend rundev e2e
|
||||
e2e-down:
|
||||
docker compose -f compose-e2e-test.yml down -v
|
||||
|
||||
# the suite brings the stack up itself when it finds none, so e2e-up is only worth running
|
||||
# to keep the containers between invocations
|
||||
e2e:
|
||||
cd e2e && go test -tags=e2e -count 1 -timeout 20m ./...
|
||||
|
||||
e2e-ui:
|
||||
cd e2e && E2E_HEADLESS=false E2E_KEEP=1 go test -tags=e2e -count 1 -v -timeout 20m ./...
|
||||
|
||||
.PHONY: bin docker dockerx release race_test backend frontend rundev e2e e2e-up e2e-down e2e-ui
|
||||
|
||||
@@ -20,9 +20,6 @@ linters:
|
||||
- unparam
|
||||
- unused
|
||||
settings:
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
gosec:
|
||||
excludes:
|
||||
- G117 # false positive: struct field name matches "secret" pattern
|
||||
@@ -38,8 +35,6 @@ linters:
|
||||
govet:
|
||||
enable:
|
||||
- shadow
|
||||
lll:
|
||||
line-length: 140
|
||||
misspell:
|
||||
locale: US
|
||||
exclusions:
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
../site/src/docs/contributing/backend/index.md
|
||||
../site/content/docs/contributing/backend/index.md
|
||||
@@ -3,10 +3,10 @@ module github.com/umputun/remark42/memory_store
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/go-pkgz/jrpc v0.4.0
|
||||
github.com/go-pkgz/lgr v0.12.3
|
||||
github.com/go-pkgz/jrpc v0.4.2
|
||||
github.com/go-pkgz/lgr v0.12.4
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/umputun/remark42/backend v1.1000.0
|
||||
)
|
||||
|
||||
@@ -16,25 +16,19 @@ require (
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
|
||||
github.com/go-pkgz/rest v1.22.0 // indirect
|
||||
github.com/go-pkgz/routegroup v1.6.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 // indirect
|
||||
github.com/go-pkgz/rest v1.24.0 // indirect
|
||||
github.com/go-pkgz/routegroup v1.6.1 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/image v0.43.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/image v0.45.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/umputun/remark42/backend v1.1000.0 => ../../
|
||||
|
||||
@@ -12,60 +12,41 @@ github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q3
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY=
|
||||
github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo=
|
||||
github.com/go-pkgz/lgr v0.12.3 h1:QDug7kRkEsuQtruT9fNF5PVT2kZUqCDPc4GmsgS3fP8=
|
||||
github.com/go-pkgz/lgr v0.12.3/go.mod h1:lpCDgVvCIxBHZp8+sGCj9MPctIzKZyZ3QdE19ddqd54=
|
||||
github.com/go-pkgz/rest v1.22.0 h1:d3XFKlmAGBiU9MQER9/n46iXpyUr8IQUtfjU8JlqkkY=
|
||||
github.com/go-pkgz/rest v1.22.0/go.mod h1:+AHzjHazq7Z3Tk/kRWOhbbAz/YZlUV40feC1Hf4NtbE=
|
||||
github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4=
|
||||
github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU=
|
||||
github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ=
|
||||
github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok=
|
||||
github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8=
|
||||
github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI=
|
||||
github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc=
|
||||
github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q=
|
||||
github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
|
||||
github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
|
||||
golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
|
||||
@@ -115,14 +115,17 @@ func TestRPC_imgCleanupHndl(t *testing.T) {
|
||||
assert.Equal(t, 1462, len(img))
|
||||
assert.Equal(t, gopherPNGBytes(), img)
|
||||
|
||||
// wait for image to expire
|
||||
time.Sleep(time.Millisecond * 50)
|
||||
// reset the time to cleanup
|
||||
// age the image past the ttl used below, so the reset that follows is what keeps it on
|
||||
// staging rather than the image simply being young
|
||||
const stagingTTL = 500 * time.Millisecond
|
||||
time.Sleep(stagingTTL + 100*time.Millisecond)
|
||||
|
||||
// reset the time to cleanup, which leaves a full ttl before it could be collected again
|
||||
err = ri.ResetCleanupTimer(id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// cleanup, should not affect the new image
|
||||
err = ri.Cleanup(context.TODO(), time.Millisecond*45)
|
||||
err = ri.Cleanup(context.TODO(), stagingTTL)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// load after cleanup should succeed
|
||||
|
||||
@@ -8,7 +8,6 @@ package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
@@ -20,27 +19,31 @@ import (
|
||||
"github.com/umputun/remark42/memory_store/accessor"
|
||||
)
|
||||
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for range 10 {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
return port
|
||||
}
|
||||
|
||||
func waitForHTTPServerStart(port int) {
|
||||
// wait for up to 3 seconds for server to start before returning it
|
||||
// waitForHTTPServerStart blocks until the server on port answers, failing the test naming the
|
||||
// port if it never does
|
||||
func waitForHTTPServerStart(t *testing.T, port int) {
|
||||
t.Helper()
|
||||
client := http.Client{Timeout: time.Second}
|
||||
for range 300 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
defer client.CloseIdleConnections()
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}, 30*time.Second, 10*time.Millisecond, "http server on port %d didn't start", port)
|
||||
}
|
||||
|
||||
func prepTestStore(t *testing.T) (port int, teardown func()) {
|
||||
@@ -61,14 +64,17 @@ func prepTestStore(t *testing.T) (port int, teardown func()) {
|
||||
admRecDisabled.Enabled = false
|
||||
adm.Set("test-site-disabled", admRecDisabled)
|
||||
|
||||
port = chooseRandomUnusedPort()
|
||||
port = chooseUnusedPort(t)
|
||||
go func() {
|
||||
_ = s.Run(port)
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
return port, func() {
|
||||
// every test client here uses http.DefaultTransport, so their keep-alive connections
|
||||
// sit in one shared pool; Shutdown waits on them and hits its own 5s deadline otherwise
|
||||
http.DefaultTransport.(*http.Transport).CloseIdleConnections()
|
||||
require.NoError(t, s.Shutdown())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/jessevdk/go-flags"
|
||||
@@ -133,15 +132,14 @@ func TestImport_ExecuteFailed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestImport_ExecuteTimeout(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
fmt.Fprintln(w, "some response")
|
||||
fmt.Fprintln(w, string(body))
|
||||
// hold the response until the client gives up on its own timeout
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ type ServerCommand struct {
|
||||
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
|
||||
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
|
||||
|
||||
SendJWTHeader bool `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"send JWT as a header instead of server-set cookie; with this enabled, frontend stores the JWT in a client-side cookie (note: increases vulnerability to XSS attacks)"`
|
||||
SendJWTHeader bool `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"also send JWT as a header, so the frontend can store it in a client-side cookie that survives third-party cookie blocking; server-set cookies are still sent (note: increases vulnerability to XSS attacks)"`
|
||||
SameSite string `long:"same-site" env:"SAME_SITE" description:"set same site policy for cookies" choice:"default" choice:"none" choice:"lax" choice:"strict" default:"default"` // nolint
|
||||
|
||||
Apple AppleGroup `group:"apple" namespace:"apple" env-namespace:"APPLE" description:"Apple OAuth"`
|
||||
@@ -252,6 +252,7 @@ type TelegramGroup struct {
|
||||
type SMTPGroup struct {
|
||||
Host string `long:"host" env:"HOST" description:"SMTP host"`
|
||||
Port int `long:"port" env:"PORT" description:"SMTP port"`
|
||||
HELOHost string `long:"helo_host" env:"HELO_HOST" description:"SMTP HELO/EHLO hostname"`
|
||||
Username string `long:"username" env:"USERNAME" description:"SMTP user name"`
|
||||
Password string `long:"password" env:"PASSWORD" description:"SMTP password"`
|
||||
TLS bool `long:"tls" env:"TLS" description:"enable TLS"`
|
||||
@@ -1166,6 +1167,7 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
|
||||
params := sender.EmailParams{
|
||||
Host: s.SMTP.Host,
|
||||
Port: s.SMTP.Port,
|
||||
HELOHost: s.SMTP.HELOHost,
|
||||
SMTPUserName: s.SMTP.Username,
|
||||
SMTPPassword: s.SMTP.Password,
|
||||
TimeOut: s.SMTP.TimeOut,
|
||||
@@ -1322,6 +1324,7 @@ func (s *ServerCommand) makeNotifyDestinations(authenticator *auth.Service) ([]n
|
||||
smtpParams := ntf.SMTPParams{
|
||||
Host: s.SMTP.Host,
|
||||
Port: s.SMTP.Port,
|
||||
HELOHost: s.SMTP.HELOHost,
|
||||
TLS: s.SMTP.TLS,
|
||||
StartTLS: s.SMTP.StartTLS,
|
||||
InsecureSkipVerify: s.SMTP.InsecureSkipVerify,
|
||||
|
||||
+239
-91
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -25,15 +24,33 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
// budget for a server to bind and answer, generous enough for a loaded CI runner
|
||||
serverStartTimeout = 30 * time.Second
|
||||
serverStartPoll = 10 * time.Millisecond
|
||||
|
||||
// budget for a server to stop once asked. tight enough to catch a shutdown that hangs,
|
||||
// loose enough not to depend on how loaded the runner is
|
||||
serverStopTimeout = 10 * time.Second
|
||||
|
||||
// connect budget for a single probe. kept off the poll interval so a slow loopback connect
|
||||
// on a loaded runner does not look like a server that is not listening
|
||||
probeDialTimeout = time.Second
|
||||
|
||||
// the /auth/ group is limited to 2 req/s, so retries sit at its refill interval rather than
|
||||
// above it, which would only manufacture more 429s
|
||||
authRetryPoll = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
func TestServerApp(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
return o
|
||||
})
|
||||
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
// send ping
|
||||
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
|
||||
@@ -68,7 +85,7 @@ func TestServerApp(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerApp_DevMode(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
o.AdminPasswd = "password"
|
||||
@@ -77,7 +94,7 @@ func TestServerApp_DevMode(t *testing.T) {
|
||||
})
|
||||
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
providers := app.restSrv.Authenticator.Providers()
|
||||
require.Equal(t, 11+1, len(providers), "extra auth provider")
|
||||
@@ -97,7 +114,7 @@ func TestServerApp_DevMode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerApp_CustomOAuthProvider(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
o.Auth.Custom.Name = "oidc"
|
||||
@@ -110,7 +127,7 @@ func TestServerApp_CustomOAuthProvider(t *testing.T) {
|
||||
})
|
||||
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
providers := app.restSrv.Authenticator.Providers()
|
||||
require.Equal(t, 11+1, len(providers), "extra auth provider")
|
||||
@@ -121,7 +138,7 @@ func TestServerApp_CustomOAuthProvider(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerApp_AnonMode(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
o.Auth.Anonymous = true
|
||||
@@ -129,7 +146,7 @@ func TestServerApp_AnonMode(t *testing.T) {
|
||||
})
|
||||
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
providers := app.restSrv.Authenticator.Providers()
|
||||
require.Equal(t, 11+1, len(providers), "extra auth provider for anon")
|
||||
@@ -148,8 +165,7 @@ func TestServerApp_AnonMode(t *testing.T) {
|
||||
assert.Equal(t, "pong", string(body))
|
||||
|
||||
// try to login with good name
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
@@ -168,57 +184,43 @@ func TestServerApp_AnonMode(t *testing.T) {
|
||||
assert.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
// try to login with non-latin name
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
nonLatin := fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port)
|
||||
resp = getRetryThrottled(t, &client, nonLatin)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
// try to login with bad name
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
// try to login with short name
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
// try to login with name what have space in prefix
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
// try to login with name what have space in suffix
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
// try to login with long name
|
||||
time.Sleep(time.Second)
|
||||
ln := strings.Repeat("x", 65)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
// try to login with admin name
|
||||
time.Sleep(time.Second)
|
||||
resp, err = client.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
|
||||
require.NoError(t, err)
|
||||
resp = getRetryThrottled(t, &client, fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
// try to add a comment as anonymous with admin name
|
||||
time.Sleep(time.Second)
|
||||
req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment?site=remark", port),
|
||||
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
|
||||
require.NoError(t, err)
|
||||
@@ -250,12 +252,12 @@ func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (tkn s
|
||||
|
||||
func TestServerApp_WithSSL(t *testing.T) {
|
||||
opts := ServerCommand{}
|
||||
sslPort := chooseRandomUnusedPort()
|
||||
sslPort := chooseUnusedPort(t)
|
||||
opts.SetCommon(CommonOpts{RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort), SharedSecret: "123456"})
|
||||
|
||||
// prepare options
|
||||
p := flags.NewParser(&opts, flags.Default)
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=" + strconv.Itoa(port), "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
|
||||
"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db",
|
||||
"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
|
||||
@@ -270,8 +272,9 @@ func TestServerApp_WithSSL(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel() // this context is not the one createAppFromCmd registers for cleanup
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPSServerStart(sslPort)
|
||||
waitForServerStart(t, sslPort, port) // the redirect check below uses the plain http port
|
||||
|
||||
client := http.Client{
|
||||
// prevent http redirect
|
||||
@@ -312,7 +315,7 @@ func TestServerApp_WithRemote(t *testing.T) {
|
||||
|
||||
// prepare options
|
||||
p := flags.NewParser(&opts, flags.Default)
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
|
||||
"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
|
||||
"--port=" + strconv.Itoa(port), "--avatar.fs.path=/tmp",
|
||||
@@ -326,8 +329,9 @@ func TestServerApp_WithRemote(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel() // this context is not the one createAppFromCmd registers for cleanup
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
// send ping
|
||||
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
|
||||
@@ -516,34 +520,117 @@ func TestServerApp_InvalidCustomOAuthProviderName(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerApp_Shutdown(t *testing.T) {
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = chooseRandomUnusedPort()
|
||||
o.Port = port
|
||||
return o
|
||||
})
|
||||
time.AfterFunc(100*time.Millisecond, func() {
|
||||
cancel()
|
||||
})
|
||||
st := time.Now()
|
||||
err := app.run(ctx)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, time.Since(st).Seconds() < 1, "should take about 100msec")
|
||||
|
||||
// cancel once the server actually answers, so the test measures shutdown and not startup.
|
||||
// the deferred cancel also covers a failed wait, keeping app.run from racing the next test
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- app.run(ctx) }()
|
||||
defer cancel()
|
||||
waitForHTTPServerStart(t, port)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
assert.NoError(t, err)
|
||||
case <-time.After(serverStopTimeout):
|
||||
t.Fatal("server app did not stop after context cancel")
|
||||
}
|
||||
app.Wait()
|
||||
}
|
||||
|
||||
func TestServerApp_MainSignal(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
<-done
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
|
||||
// TestServerApp_ClaimsUpd covers the hook the authenticator runs on every token mint, refresh
|
||||
// included: it stamps admin, blocked and email onto the claims and blocks impersonation of a
|
||||
// restricted name. Calling the updater directly keeps it independent of when a token expires.
|
||||
func TestServerApp_ClaimsUpd(t *testing.T) {
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
return o
|
||||
})
|
||||
|
||||
// the app owns stores and services that only run closes, so it goes through the usual
|
||||
// lifecycle here rather than being built and abandoned
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(t, port)
|
||||
defer app.Wait()
|
||||
defer cancel()
|
||||
|
||||
upd := app.restSrv.Authenticator.TokenService().ClaimsUpd
|
||||
require.NotNil(t, upd, "claims updater wired into the token service")
|
||||
|
||||
claimsFor := func(id, name string) token.Claims {
|
||||
return token.Claims{
|
||||
RegisteredClaims: jwt.RegisteredClaims{Audience: jwt.ClaimStrings{"remark"}},
|
||||
User: &token.User{ID: id, Name: name},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("plain user gets no attributes", func(t *testing.T) {
|
||||
res := upd.Update(claimsFor("provider1_dev", "developer"))
|
||||
assert.False(t, res.User.IsAdmin(), "not an admin")
|
||||
assert.False(t, res.User.BoolAttr("blocked"), "not blocked")
|
||||
assert.Empty(t, res.User.Email, "no email on file")
|
||||
})
|
||||
|
||||
t.Run("admin from the admin store", func(t *testing.T) {
|
||||
res := upd.Update(claimsFor("id1", "admin one"))
|
||||
assert.True(t, res.User.IsAdmin(), "id1 is listed as admin")
|
||||
})
|
||||
|
||||
t.Run("blocked user carries the blocked attribute", func(t *testing.T) {
|
||||
require.NoError(t, app.restSrv.DataService.SetBlock("remark", "blocked_user", true, time.Hour))
|
||||
res := upd.Update(claimsFor("blocked_user", "blocked"))
|
||||
assert.True(t, res.User.BoolAttr("blocked"), "block is reflected on refresh")
|
||||
})
|
||||
|
||||
t.Run("email is read from the store", func(t *testing.T) {
|
||||
_, err := app.restSrv.DataService.SetUserEmail("remark", "with_email", "user@example.com")
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
res := upd.Update(claimsFor("with_email", "someone"))
|
||||
assert.Equal(t, "user@example.com", res.User.Email)
|
||||
})
|
||||
|
||||
t.Run("anonymous impersonating a restricted name is blocked", func(t *testing.T) {
|
||||
res := upd.Update(claimsFor("anonymous_x", " UmpUtun "))
|
||||
assert.True(t, res.User.BoolAttr("blocked"), "restricted name matched case and space insensitively")
|
||||
})
|
||||
|
||||
t.Run("email user impersonating a restricted name is blocked", func(t *testing.T) {
|
||||
res := upd.Update(claimsFor("email_x", "bobuk"))
|
||||
assert.True(t, res.User.BoolAttr("blocked"))
|
||||
})
|
||||
|
||||
t.Run("regular user may carry a restricted name", func(t *testing.T) {
|
||||
res := upd.Update(claimsFor("provider1_someone", "umputun"))
|
||||
assert.False(t, res.User.BoolAttr("blocked"), "only anonymous and email logins are checked")
|
||||
})
|
||||
|
||||
t.Run("claims without a user pass through", func(t *testing.T) {
|
||||
res := upd.Update(token.Claims{RegisteredClaims: jwt.RegisteredClaims{Audience: jwt.ClaimStrings{"remark"}}})
|
||||
assert.Nil(t, res.User)
|
||||
})
|
||||
|
||||
t.Run("claims without exactly one audience pass through", func(t *testing.T) {
|
||||
c := claimsFor("id1", "admin one")
|
||||
c.Audience = jwt.ClaimStrings{"remark", "second"}
|
||||
res := upd.Update(c)
|
||||
assert.False(t, res.User.IsAdmin(), "attributes need a single audience to resolve the site")
|
||||
})
|
||||
}
|
||||
|
||||
func TestServerApp_MainSignal(t *testing.T) {
|
||||
sigErr := make(chan error, 1)
|
||||
|
||||
s := ServerCommand{}
|
||||
s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
|
||||
|
||||
p := flags.NewParser(&s, flags.Default)
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
|
||||
"--avatar.bolt.file=/tmp/ava-test.db", "--port=" + strconv.Itoa(port), "--image.fs.path=/tmp"}
|
||||
defer os.Remove("/tmp/xyz")
|
||||
@@ -551,15 +638,26 @@ func TestServerApp_MainSignal(t *testing.T) {
|
||||
defer os.Remove("/tmp/ava-test.db")
|
||||
_, err := p.ParseArgs(args)
|
||||
require.NoError(t, err)
|
||||
st := time.Now()
|
||||
close(done)
|
||||
// the signal goes out only once the server answers: SIGTERM landing before the handler is
|
||||
// installed kills the test process, so a wait that timed out reports instead of sending it
|
||||
go func() {
|
||||
started := waitForServerPort(port, serverStartTimeout)
|
||||
// signal either way: Execute blocks until it gets one, so bailing out here would hang
|
||||
// the test until the package timeout instead of failing with the reason
|
||||
killErr := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
|
||||
if !started {
|
||||
killErr = fmt.Errorf("server on port %d didn't start", port)
|
||||
}
|
||||
sigErr <- killErr
|
||||
}()
|
||||
|
||||
err = s.Execute(args)
|
||||
assert.NoError(t, err, "execute should be without errors")
|
||||
assert.True(t, time.Since(st).Seconds() < 5, "should take under five sec", time.Since(st).Seconds())
|
||||
require.NoError(t, <-sigErr, "SIGTERM not delivered")
|
||||
}
|
||||
|
||||
func TestServerApp_RunCanceledBeforeRESTStart(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
return o
|
||||
@@ -569,17 +667,19 @@ func TestServerApp_RunCanceledBeforeRESTStart(t *testing.T) {
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- app.run(ctx) }()
|
||||
|
||||
// the budget is generous on purpose: the assertion is that run exits rather than hangs, and
|
||||
// store construction can take a while on a loaded runner
|
||||
select {
|
||||
case err := <-errCh:
|
||||
require.NoError(t, err)
|
||||
app.Wait()
|
||||
case <-time.After(time.Second):
|
||||
waitForHTTPServerStart(port)
|
||||
case <-time.After(serverStartTimeout):
|
||||
waitForHTTPServerStart(t, port)
|
||||
app.restSrv.Shutdown()
|
||||
select {
|
||||
case <-errCh:
|
||||
app.Wait()
|
||||
case <-time.After(time.Second):
|
||||
case <-time.After(serverStartTimeout):
|
||||
t.Fatal("server app did not stop after forced REST shutdown")
|
||||
}
|
||||
t.Fatal("server app should exit when context is canceled before REST server starts")
|
||||
@@ -747,24 +847,25 @@ func Test_ACMEEmail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerAuthHooks(t *testing.T) {
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
app, ctx, cancel := prepServerApp(t, func(o ServerCommand) ServerCommand {
|
||||
o.Port = port
|
||||
return o
|
||||
})
|
||||
|
||||
go func() { _ = app.run(ctx) }()
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
// make a token for user dev
|
||||
// make a token for user dev. nothing here checks expiry, so the lifetime only has to
|
||||
// outlast the whole test
|
||||
tkService := app.restSrv.Authenticator.TokenService()
|
||||
tkService.TokenDuration = time.Second
|
||||
tkService.TokenDuration = time.Hour
|
||||
|
||||
claims := token.Claims{
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
Audience: jwt.ClaimStrings{"remark"},
|
||||
Issuer: "remark",
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second)),
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)),
|
||||
NotBefore: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)),
|
||||
},
|
||||
User: &token.User{
|
||||
@@ -867,8 +968,7 @@ func TestServerAuthHooks(t *testing.T) {
|
||||
body, err = io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
|
||||
"blocked user can't post, \n"+tk+"\n"+string(body))
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode, "blocked user can't post, \n"+tk+"\n"+string(body))
|
||||
|
||||
cancel()
|
||||
app.Wait()
|
||||
@@ -968,40 +1068,79 @@ func Test_getAllowedRedirectHosts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for range 10 {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
return port
|
||||
}
|
||||
|
||||
func waitForHTTPServerStart(port int) {
|
||||
// wait for up to 3 seconds for server to start before returning it
|
||||
// waitForHTTPServerStart blocks until the server on port answers, failing the test naming the
|
||||
// port if it never does
|
||||
func waitForHTTPServerStart(t *testing.T, port int) {
|
||||
t.Helper()
|
||||
client := http.Client{Timeout: time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
for range 300 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}, serverStartTimeout, serverStartPoll, "http server on port %d didn't start", port)
|
||||
}
|
||||
|
||||
// waitForServerStart blocks until something accepts on every listed port, failing the test
|
||||
// naming the port that never came up
|
||||
func waitForServerStart(t *testing.T, ports ...int) {
|
||||
t.Helper()
|
||||
for _, port := range ports {
|
||||
require.True(t, waitForServerPort(port, serverStartTimeout), "server on port %d didn't start", port)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForHTTPSServerStart(port int) {
|
||||
// wait for up to 3 seconds for HTTPS server to start
|
||||
for range 300 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
|
||||
if conn != nil {
|
||||
_ = conn.Close()
|
||||
break
|
||||
// getRetryThrottled issues a GET and retries while the auth routes answer 429, since the /auth/
|
||||
// group is limited to 2 req/s and this test logs in more often than that. a transport error is
|
||||
// retried a couple of times and then reported as itself, so a dead server is not read as throttling
|
||||
func getRetryThrottled(t *testing.T, client *http.Client, url string) *http.Response {
|
||||
t.Helper()
|
||||
const transportRetries = 2
|
||||
errCount := 0
|
||||
for deadline := time.Now().Add(serverStartTimeout); time.Now().Before(deadline); time.Sleep(authRetryPoll) {
|
||||
r, err := client.Get(url)
|
||||
if err != nil {
|
||||
errCount++
|
||||
require.LessOrEqual(t, errCount, transportRetries, "request to %s failed: %v", url, err)
|
||||
continue
|
||||
}
|
||||
if r.StatusCode == http.StatusTooManyRequests {
|
||||
_ = r.Body.Close()
|
||||
continue
|
||||
}
|
||||
return r
|
||||
}
|
||||
t.Fatalf("request to %s kept being rate limited", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForServerPort blocks until something accepts on port, reporting whether it came up.
|
||||
// unlike the require-based helpers it is safe to call off the test goroutine.
|
||||
func waitForServerPort(port int, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), probeDialTimeout)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
time.Sleep(serverStartPoll)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
|
||||
@@ -1064,6 +1203,9 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
|
||||
|
||||
func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// a require in a readiness wait exits the test goroutine, so without this an app started in
|
||||
// a goroutine would never be stopped and goleak would report it instead of the failure
|
||||
t.Cleanup(cancel)
|
||||
app, err := cmd.newServerApp(ctx)
|
||||
require.NoError(t, err)
|
||||
return app, ctx, cancel
|
||||
@@ -1073,8 +1215,14 @@ func TestMain(m *testing.M) {
|
||||
// ignore is added only for GitHub Actions, can't reproduce locally
|
||||
goleak.VerifyTestMain(
|
||||
m,
|
||||
// the shutdown goroutine in serverApp.run is not joined by Wait, and Rest.Shutdown gives
|
||||
// httpServer.Shutdown a second, which can outlast goleak's retry budget on a loaded runner
|
||||
goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
|
||||
// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
|
||||
goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
|
||||
// regexp2, pulled in by chroma for syntax highlighting, keeps one shared clock goroutine
|
||||
// alive for up to a second after the last match with a timeout, sleeping in 100ms ticks.
|
||||
// it ends on its own, but a binary that finishes inside that window is reported as leaking
|
||||
goleak.IgnoreAnyFunction("github.com/dlclark/regexp2/v2.runClock"),
|
||||
)
|
||||
}
|
||||
|
||||
+34
-26
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -25,7 +24,7 @@ func Test_Main(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
|
||||
"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
|
||||
|
||||
@@ -48,7 +47,7 @@ func Test_Main(t *testing.T) {
|
||||
<-finished
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
@@ -63,9 +62,9 @@ func TestMain_WithWebhook(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
var webhookSent int32
|
||||
var webhookSent atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
atomic.StoreInt32(&webhookSent, 1)
|
||||
webhookSent.Store(1)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
|
||||
b, e := io.ReadAll(r.Body)
|
||||
@@ -76,7 +75,7 @@ func TestMain_WithWebhook(t *testing.T) {
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
|
||||
"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg",
|
||||
"--admin-passwd=password", "--site=remark", "--notify.admins=webhook"}
|
||||
@@ -107,7 +106,7 @@ func TestMain_WithWebhook(t *testing.T) {
|
||||
<-finished
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(port)
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
resp, err := http.Post(fmt.Sprintf("http://admin:password@localhost:%d/api/v1/comment", port), "",
|
||||
strings.NewReader(`{"text": "env test", "locator":{"url": "https://radio-t.com", "site": "remark"}}`))
|
||||
@@ -117,8 +116,8 @@ func TestMain_WithWebhook(t *testing.T) {
|
||||
|
||||
// wait for webhook to be sent before shutting down
|
||||
assert.Eventually(t, func() bool {
|
||||
return atomic.LoadInt32(&webhookSent) == int32(1)
|
||||
}, time.Second, 100*time.Millisecond, "webhook was not sent")
|
||||
return webhookSent.Load() == int32(1)
|
||||
}, 30*time.Second, 10*time.Millisecond, "webhook was not sent")
|
||||
}
|
||||
|
||||
func TestGetDump(t *testing.T) {
|
||||
@@ -129,37 +128,46 @@ func TestGetDump(t *testing.T) {
|
||||
t.Logf("\n dump: %s", dump)
|
||||
}
|
||||
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for range 10 {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
return port
|
||||
}
|
||||
|
||||
func waitForHTTPServerStart(port int) {
|
||||
// wait for up to 10 seconds for server to start before returning it
|
||||
// waitForHTTPServerStart blocks until the server on port answers, failing the test naming the
|
||||
// port if it never does
|
||||
func waitForHTTPServerStart(t *testing.T, port int) {
|
||||
t.Helper()
|
||||
client := http.Client{Timeout: time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
for range 100 {
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}, 30*time.Second, 10*time.Millisecond, "http server on port %d didn't start", port)
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// both ignores are for leaks which are detected locally
|
||||
goleak.VerifyTestMain(
|
||||
m,
|
||||
goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
|
||||
// the shutdown goroutine in serverApp.run is not joined by Wait, and Rest.Shutdown gives
|
||||
// httpServer.Shutdown a second, which can outlast goleak's retry budget on a loaded runner
|
||||
goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
|
||||
goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
|
||||
// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
|
||||
goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
|
||||
// regexp2, pulled in by chroma for syntax highlighting, keeps one shared clock goroutine
|
||||
// alive for up to a second after the last match with a timeout, sleeping in 100ms ticks.
|
||||
// it ends on its own, but a binary that finishes inside that window is reported as leaking
|
||||
goleak.IgnoreAnyFunction("github.com/dlclark/regexp2/v2.runClock"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -50,9 +51,7 @@ func TestBackup_MakeBackup(t *testing.T) {
|
||||
expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
|
||||
assert.Equal(t, expFile, fname)
|
||||
|
||||
fi, err := os.Lstat(expFile)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(52), fi.Size())
|
||||
assert.Equal(t, exportedPayload, gzContent(t, expFile))
|
||||
}
|
||||
|
||||
func TestBackup_Do(t *testing.T) {
|
||||
@@ -71,15 +70,31 @@ func TestBackup_Do(t *testing.T) {
|
||||
bk.Do(ctx)
|
||||
|
||||
expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
|
||||
fi, err := os.Lstat(expFile)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(52), fi.Size())
|
||||
assert.Equal(t, exportedPayload, gzContent(t, expFile))
|
||||
})
|
||||
}
|
||||
|
||||
const exportedPayload = "some export blah blah 1234567890"
|
||||
|
||||
// the compressed size is not assertable: it moves with the compress/flate version
|
||||
func gzContent(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
fh, err := os.Open(name) //nolint:gosec // path is built by the test
|
||||
require.NoError(t, err)
|
||||
defer func() { assert.NoError(t, fh.Close()) }()
|
||||
|
||||
gz, err := gzip.NewReader(fh)
|
||||
require.NoError(t, err)
|
||||
defer func() { assert.NoError(t, gz.Close()) }()
|
||||
|
||||
b, err := io.ReadAll(gz)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
type mockExporter struct{}
|
||||
|
||||
func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
|
||||
_, err := w.Write([]byte("some export blah blah 1234567890"))
|
||||
_, err := w.Write([]byte(exportedPayload))
|
||||
return 1000, err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
@@ -46,8 +47,8 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
|
||||
|
||||
log.Printf("[DEBUG] exporting %d topics", len(topics))
|
||||
commentsCount := 0
|
||||
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
|
||||
topic := topics[i]
|
||||
for _, topic := range slices.Backward(topics) { // topics from List sorted in opposite direction
|
||||
|
||||
comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
|
||||
if e != nil {
|
||||
return commentsCount, e
|
||||
|
||||
@@ -3,6 +3,7 @@ package notify
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/url"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/go-pkgz/repeater/v2"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/templates"
|
||||
@@ -160,23 +160,23 @@ func (e *Email) Send(ctx context.Context, req Request) error {
|
||||
default:
|
||||
}
|
||||
|
||||
result := new(multierror.Error)
|
||||
var errs []error
|
||||
|
||||
for _, email := range req.Emails {
|
||||
err := e.buildAndSendMessage(ctx, req, email, false)
|
||||
if err != nil {
|
||||
result = multierror.Append(fmt.Errorf("problem sending user email notification to %q: %w", email, err))
|
||||
errs = append(errs, fmt.Errorf("problem sending user email notification to %q: %w", email, err))
|
||||
}
|
||||
}
|
||||
|
||||
for _, email := range e.AdminEmails {
|
||||
err := e.buildAndSendMessage(ctx, req, email, true)
|
||||
if err != nil {
|
||||
result = multierror.Append(fmt.Errorf("problem sending admin email notification to %q: %w", email, err))
|
||||
errs = append(errs, fmt.Errorf("problem sending admin email notification to %q: %w", email, err))
|
||||
}
|
||||
}
|
||||
|
||||
return result.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (e *Email) buildAndSendMessage(ctx context.Context, req Request, email string, forAdmin bool) error {
|
||||
|
||||
@@ -110,10 +110,10 @@ func TestEmailSendErrors(t *testing.T) {
|
||||
e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
|
||||
"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\": "+
|
||||
"problem sending user email notification to \"bad@example.org\": "+
|
||||
"error executing template to build comment reply message: "+
|
||||
"template: test:1:2: executing \"test\" at <.Test>: "+
|
||||
"can't evaluate field Test in type notify.msgTmplData\n\n")
|
||||
"can't evaluate field Test in type notify.msgTmplData")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
@@ -121,8 +121,14 @@ func TestEmailSendErrors(t *testing.T) {
|
||||
"sending email messages about comment \"999\" aborted due to canceled context")
|
||||
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad@example.org"}}),
|
||||
"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\":"+
|
||||
" error creating token for unsubscribe link: token generation error\n\n")
|
||||
"problem sending user email notification to \"bad@example.org\":"+
|
||||
" error creating token for unsubscribe link: token generation error")
|
||||
|
||||
// errors for all failed recipients are reported, not just the last one
|
||||
assert.EqualError(t, e.Send(context.Background(),
|
||||
Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad1@example.org", "bad2@example.org"}}),
|
||||
"problem sending user email notification to \"bad1@example.org\": error creating token for unsubscribe link: token generation error\n"+
|
||||
"problem sending user email notification to \"bad2@example.org\": error creating token for unsubscribe link: token generation error")
|
||||
}
|
||||
|
||||
func TestEmailSend_ExitConditions(t *testing.T) {
|
||||
|
||||
@@ -19,7 +19,7 @@ type Service struct {
|
||||
queue chan Request
|
||||
verificationQueue chan VerificationRequest
|
||||
|
||||
closed uint32 // non-zero means closed. uses uint instead of bool for atomic
|
||||
closed atomic.Uint32 // non-zero means closed. uses uint instead of bool for atomic
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
|
||||
|
||||
// Submit Request to internal channel if not busy, drop if can't send
|
||||
func (s *Service) Submit(req Request) {
|
||||
if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
|
||||
if len(s.destinations) == 0 || s.closed.Load() != 0 {
|
||||
return
|
||||
}
|
||||
if s.dataService != nil && req.Comment.ParentID != "" {
|
||||
@@ -130,7 +130,7 @@ func (s *Service) getNotificationTargets(
|
||||
|
||||
// SubmitVerification to internal channel if not busy, drop if can't send
|
||||
func (s *Service) SubmitVerification(req VerificationRequest) {
|
||||
if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
|
||||
if len(s.destinations) == 0 || s.closed.Load() != 0 {
|
||||
return
|
||||
}
|
||||
select {
|
||||
@@ -155,7 +155,7 @@ func (s *Service) Close() {
|
||||
s.cancel()
|
||||
<-s.ctx.Done()
|
||||
}
|
||||
atomic.StoreUint32(&s.closed, 1)
|
||||
s.closed.Store(1)
|
||||
}
|
||||
|
||||
func (s *Service) do() {
|
||||
|
||||
@@ -15,10 +15,14 @@ type MockDest struct {
|
||||
id int
|
||||
closed bool
|
||||
lock sync.Mutex
|
||||
block chan struct{} // if non-nil, Send/SendVerification wait on it before recording, letting tests pin the consumer
|
||||
}
|
||||
|
||||
// Send mock
|
||||
func (m *MockDest) Send(ctx context.Context, r Request) error {
|
||||
if m.block != nil {
|
||||
<-m.block
|
||||
}
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -33,6 +37,9 @@ func (m *MockDest) Send(ctx context.Context, r Request) error {
|
||||
|
||||
// SendVerification mock
|
||||
func (m *MockDest) SendVerification(ctx context.Context, v VerificationRequest) error {
|
||||
if m.block != nil {
|
||||
<-m.block
|
||||
}
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
|
||||
@@ -2,7 +2,6 @@ package notify
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
|
||||
@@ -49,26 +48,38 @@ func TestService_WithDestinations(t *testing.T) {
|
||||
|
||||
func TestService_WithDrops(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
// gated destinations pin the consumer on the first item so the size-1 queue
|
||||
// fills deterministically and the overflow is dropped regardless of scheduling
|
||||
gate := make(chan struct{})
|
||||
d1, d2 := &MockDest{id: 1, block: gate}, &MockDest{id: 2, block: gate}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}})
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}})
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}})
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}}) // consumed, consumer blocks in Send on the gate
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}}) // fills the size-1 queue
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}}) // queue full, dropped
|
||||
synctest.Wait()
|
||||
|
||||
close(gate) // release the consumer: it finishes 100 then processes 101
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
|
||||
|
||||
assert.LessOrEqual(t, len(d1.Get()), 2, "at least one comment from three dropped from d1, got: %v", d1.Get())
|
||||
assert.LessOrEqual(t, len(d2.Get()), 2, "at least one comment from three dropped from d2, got: %v", d2.Get())
|
||||
require.Len(t, d1.Get(), 2, "one comment of three dropped from d1, got: %v", d1.Get())
|
||||
require.Len(t, d2.Get(), 2, "one comment of three dropped from d2, got: %v", d2.Get())
|
||||
assert.Equal(t, "100", d1.Get()[0].Comment.ID)
|
||||
assert.Equal(t, "101", d1.Get()[1].Comment.ID)
|
||||
})
|
||||
}
|
||||
|
||||
func TestService_SubmitVerificationWithDrops(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
// gated destinations pin the consumer on the first item so the size-1 queue
|
||||
// fills deterministically and the overflow is dropped regardless of scheduling
|
||||
gate := make(chan struct{})
|
||||
d1, d2 := &MockDest{id: 1, block: gate}, &MockDest{id: 2, block: gate}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
@@ -77,22 +88,27 @@ func TestService_SubmitVerificationWithDrops(t *testing.T) {
|
||||
User: "testUser",
|
||||
Email: "test@example.org",
|
||||
Token: "testToken",
|
||||
})
|
||||
s.SubmitVerification(VerificationRequest{})
|
||||
s.SubmitVerification(VerificationRequest{})
|
||||
}) // consumed, consumer blocks in SendVerification on the gate
|
||||
synctest.Wait()
|
||||
s.SubmitVerification(VerificationRequest{User: "second"}) // fills the size-1 queue
|
||||
s.SubmitVerification(VerificationRequest{User: "dropped"}) // queue full, dropped
|
||||
synctest.Wait()
|
||||
|
||||
close(gate) // release the consumer: it finishes testUser then processes second
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
|
||||
s.SubmitVerification(VerificationRequest{}) // safe to send after close
|
||||
|
||||
assert.LessOrEqual(t, len(d2.GetVerify()), 2, "one request from three dropped from d2, got: %v", d2.GetVerify())
|
||||
require.Len(t, d2.GetVerify(), 2, "one request of three dropped from d2, got: %v", d2.GetVerify())
|
||||
|
||||
verifyDest := d1.GetVerify()
|
||||
require.LessOrEqual(t, len(verifyDest), 2, "one request from three dropped from d1, got: %v", verifyDest)
|
||||
require.Len(t, verifyDest, 2, "one request of three dropped from d1, got: %v", verifyDest)
|
||||
assert.Equal(t, "remark", verifyDest[0].SiteID)
|
||||
assert.Equal(t, "testUser", verifyDest[0].User)
|
||||
assert.Equal(t, "test@example.org", verifyDest[0].Email)
|
||||
assert.Equal(t, "testToken", verifyDest[0].Token)
|
||||
assert.Equal(t, "second", verifyDest[1].User)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -281,7 +297,7 @@ func TestService_Nop(t *testing.T) {
|
||||
s := NopService
|
||||
s.Submit(Request{Comment: store.Comment{}})
|
||||
s.Close()
|
||||
assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
|
||||
assert.Equal(t, uint32(1), s.closed.Load())
|
||||
}
|
||||
|
||||
type mockStore struct {
|
||||
|
||||
@@ -2,12 +2,12 @@ package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
)
|
||||
|
||||
const commentTextLengthLimit = 100
|
||||
@@ -47,14 +47,14 @@ func NewTelegram(params TelegramParams) (*Telegram, error) {
|
||||
// Send to telegram recipients
|
||||
func (t *Telegram) Send(ctx context.Context, req Request) error {
|
||||
log.Printf("[DEBUG] send telegram notification for comment ID %s", req.Comment.ID)
|
||||
result := new(multierror.Error)
|
||||
var errs []error
|
||||
|
||||
msg := t.buildMessage(req)
|
||||
|
||||
if t.AdminChannelID != "" {
|
||||
err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", t.AdminChannelID), msg)
|
||||
if err != nil {
|
||||
result = multierror.Append(result,
|
||||
errs = append(errs,
|
||||
fmt.Errorf("problem sending admin telegram notification about comment ID %s to %s: %w",
|
||||
req.Comment.ID, t.AdminChannelID, err,
|
||||
),
|
||||
@@ -66,7 +66,7 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
|
||||
for _, user := range req.Telegrams {
|
||||
err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", user), msg)
|
||||
if err != nil {
|
||||
result = multierror.Append(result,
|
||||
errs = append(errs,
|
||||
fmt.Errorf("problem sending user telegram notification about comment ID %s to %q: %w",
|
||||
req.Comment.ID, user, err,
|
||||
),
|
||||
@@ -74,7 +74,7 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// buildMessage generates message for generic notification about new comment
|
||||
|
||||
@@ -30,7 +30,6 @@ func TestTelegram_Send(t *testing.T) {
|
||||
|
||||
err := tb.Send(context.Background(), Request{Comment: c, parent: cp, Telegrams: []string{"test_user_channel"}})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "2 errors occurred")
|
||||
assert.Contains(t, err.Error(), "problem sending user telegram notification about comment ID 999 to \"test_user_channel\"")
|
||||
assert.Contains(t, err.Error(), "problem sending admin telegram notification about comment ID 999 to remark_test")
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestAdmin_Delete(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), http.NoBody)
|
||||
require.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -75,14 +75,22 @@ func TestAdmin_Delete(t *testing.T) {
|
||||
assert.Equal(t, "", cr.Text)
|
||||
assert.True(t, cr.Deleted)
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
// check last comments updated
|
||||
res, code = get(t, ts.URL+"/api/v1/last/2?site=remark42")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
comments = []store.Comment{}
|
||||
err = json.Unmarshal([]byte(res), &comments)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(comments), "should have 1 comments")
|
||||
// the last-comments list refreshes asynchronously after the delete. the polling closure runs
|
||||
// off the test goroutine, so it asserts on the CollectT it is handed rather than on t, which
|
||||
// also puts the real transport or decode error in the failure message
|
||||
pollClient := http.Client{Timeout: waitTimeout}
|
||||
defer pollClient.CloseIdleConnections()
|
||||
require.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||
lastResp, gErr := pollClient.Get(ts.URL + "/api/v1/last/2?site=remark42")
|
||||
if !assert.NoError(c, gErr) {
|
||||
return
|
||||
}
|
||||
defer lastResp.Body.Close()
|
||||
assert.Equal(c, http.StatusOK, lastResp.StatusCode)
|
||||
last := []store.Comment{}
|
||||
assert.NoError(c, json.NewDecoder(lastResp.Body).Decode(&last))
|
||||
assert.Len(c, last, 1, "should have 1 comments")
|
||||
}, waitTimeout, httpPoll)
|
||||
|
||||
// check count updated
|
||||
res, code = get(t, ts.URL+"/api/v1/count?site=remark42&url=https://radio-t.com/blah")
|
||||
@@ -139,7 +147,7 @@ func TestAdmin_Title(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/title/%s?site=remark42&url=%s/post1", ts.URL, id1, tss.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -174,7 +182,7 @@ func TestAdmin_DeleteUser(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42", ts.URL, "id2"), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
assert.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -275,7 +283,7 @@ func TestAdmin_Block(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut, url, http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
body, err = io.ReadAll(resp.Body)
|
||||
assert.NoError(t, err)
|
||||
@@ -333,10 +341,12 @@ func TestAdmin_Block(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, false, j["block"])
|
||||
|
||||
// block with ttl
|
||||
// block with ttl, checked in place rather than through another admin request, which would
|
||||
// push this test over the 10 req/s limit on that route
|
||||
makeTwoComments()
|
||||
code, _ = block(1, "50ms")
|
||||
code, _ = block(1, "500ms")
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
require.True(t, srv.adminRest.dataService.IsBlocked("remark42", "user1"), "user1 blocked with ttl")
|
||||
|
||||
// get as regular user
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
|
||||
@@ -350,7 +360,13 @@ func TestAdmin_Block(t *testing.T) {
|
||||
|
||||
srv.pubRest.cache = cache.NewScache[[]byte](cache.NewNopCache[[]byte]()) // TODO: with lru cache it won't be refreshed and invalidated for long
|
||||
// time
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// the ttl above is wide enough that the checks in between cannot outlast it, so reaching
|
||||
// here still inside the block, and the wait below observes it lapse
|
||||
require.Eventually(t, func() bool {
|
||||
return !srv.adminRest.dataService.IsBlocked("remark42", "user1")
|
||||
}, waitTimeout, pollInterval, "block with ttl did not expire")
|
||||
|
||||
res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah&sort=+time")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
comments = commentsWithInfo{}
|
||||
@@ -383,23 +399,23 @@ func TestAdmin_BlockedList(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d", ts.URL, "user1", 1), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
res, err := sendReq(t, req, adminUmputunToken)
|
||||
res, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, res.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, res.StatusCode)
|
||||
|
||||
// block user2
|
||||
// block user2 for long enough that the "two users blocked" check below cannot race the ttl
|
||||
req, err = http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d&ttl=150ms", ts.URL, "user2", 1), http.NoBody)
|
||||
fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d&ttl=1h", ts.URL, "user2", 1), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
res, err = sendReq(t, req, adminUmputunToken)
|
||||
res, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, res.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, res.StatusCode)
|
||||
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
res, err = sendReq(t, req, adminUmputunToken)
|
||||
res, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, res.StatusCode)
|
||||
users := []store.BlockedUser{}
|
||||
@@ -412,18 +428,33 @@ func TestAdmin_BlockedList(t *testing.T) {
|
||||
assert.Equal(t, "user2", users[1].ID)
|
||||
assert.Equal(t, "user2 name", users[1].Name)
|
||||
t.Logf("%+v", users)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", http.NoBody)
|
||||
// re-block user2 with a short ttl and wait for it to lapse, so the lapse is observed
|
||||
// independently of the check above
|
||||
req, err = http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/user/%s?site=remark42&block=%d&ttl=150ms", ts.URL, "user2", 1), http.NoBody)
|
||||
require.NoError(t, err)
|
||||
res, err = sendReq(t, req, adminUmputunToken)
|
||||
res, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, res.StatusCode)
|
||||
users = []store.BlockedUser{}
|
||||
err = json.NewDecoder(res.Body).Decode(&users)
|
||||
assert.NoError(t, err)
|
||||
require.NoError(t, res.Body.Close())
|
||||
assert.Equal(t, 1, len(users), "one user left blocked")
|
||||
require.Equal(t, http.StatusOK, res.StatusCode)
|
||||
|
||||
// the closure runs off the test goroutine and asserts on the CollectT it is handed, never on t
|
||||
require.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||
blockedReq, reqErr := http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", http.NoBody)
|
||||
if !assert.NoError(c, reqErr) {
|
||||
return
|
||||
}
|
||||
blockedResp, sendErr := sendReq(blockedReq, adminUmputunToken)
|
||||
if !assert.NoError(c, sendErr) {
|
||||
return
|
||||
}
|
||||
defer blockedResp.Body.Close()
|
||||
assert.Equal(c, http.StatusOK, blockedResp.StatusCode)
|
||||
blocked := []store.BlockedUser{}
|
||||
assert.NoError(c, json.NewDecoder(blockedResp.Body).Decode(&blocked))
|
||||
assert.Len(c, blocked, 1, "one user left blocked")
|
||||
}, waitTimeout, httpPoll)
|
||||
}
|
||||
|
||||
func TestAdmin_ReadOnly(t *testing.T) {
|
||||
@@ -448,11 +479,11 @@ func TestAdmin_ReadOnly(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
resp, err := sendReq(t, req, "") // non-admin user
|
||||
resp, err := sendReq(req, "") // non-admin user
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -467,7 +498,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
|
||||
assert.NoError(t, err, "can't marshal comment %+v", c)
|
||||
req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment?site=remark42", bytes.NewBuffer(b))
|
||||
require.NoError(t, err)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
@@ -476,7 +507,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
|
||||
req, err = http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=0", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -491,7 +522,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
|
||||
assert.NoError(t, err, "can't marshal comment %+v", c)
|
||||
req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment?site="+c.Locator.SiteID, bytes.NewBuffer(b))
|
||||
require.NoError(t, err)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
@@ -506,7 +537,7 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -553,7 +584,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=1", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -565,7 +596,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
|
||||
req, err = http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah&ro=0", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
@@ -594,7 +625,7 @@ func TestAdmin_Verify(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/verify/user1?site=remark42&verified=1", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -613,7 +644,7 @@ func TestAdmin_Verify(t *testing.T) {
|
||||
req, err = http.NewRequest(http.MethodPut,
|
||||
fmt.Sprintf("%s/api/v1/admin/verify/user1?site=remark42&verified=0", ts.URL), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -664,7 +695,7 @@ func TestAdmin_ExportFile(t *testing.T) {
|
||||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=remark42&mode=file", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
@@ -123,6 +123,12 @@ func ParseTrustedProxies(entries []string) ([]*net.IPNet, error) {
|
||||
// "*" and credentials enabled, rest.CORS reflects the request Origin into
|
||||
// Access-Control-Allow-Origin (rather than a literal "*"), which browsers require
|
||||
// for credentialed cross-origin requests.
|
||||
//
|
||||
// That combination is refused by default upstream, so it has to be asked for by name with
|
||||
// CorsUnsafeAnyOriginWithCredentials. The wildcard stays because the comment widget is embedded on
|
||||
// arbitrary third-party sites, which makes the set of origins unknowable. The consequence it carries
|
||||
// is that any site a signed-in user visits can read authenticated responses, so state-changing
|
||||
// requests have to keep being protected by something other than the origin, X-XSRF-Token today.
|
||||
func corsMiddleware() func(http.Handler) http.Handler {
|
||||
return R.CORS(
|
||||
R.CorsAllowedOrigins("*"),
|
||||
@@ -130,6 +136,7 @@ func corsMiddleware() func(http.Handler) http.Handler {
|
||||
R.CorsAllowedHeaders("Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"),
|
||||
R.CorsExposedHeaders("Authorization"),
|
||||
R.CorsAllowCredentials(true),
|
||||
R.CorsUnsafeAnyOriginWithCredentials(true),
|
||||
R.CorsMaxAge(300),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -50,6 +50,39 @@ func TestRouteTimeout(t *testing.T) {
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, "route without R.Timeout runs to completion")
|
||||
}
|
||||
|
||||
// TestRateLimiter covers the middleware guarding every route group: a burst past the per-second
|
||||
// allowance is refused with 429, and a client under the allowance is not. The limiter keys on
|
||||
// RemoteAddr, so the two cases use different ones rather than waiting for a bucket to refill.
|
||||
func TestRateLimiter(t *testing.T) {
|
||||
router := routegroup.New(http.NewServeMux())
|
||||
router.With(rateLimiter(1)).HandleFunc("GET /limited", func(http.ResponseWriter, *http.Request) {})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
call := func(remoteAddr string) int {
|
||||
req := httptest.NewRequest("GET", "http://example.com/limited", http.NoBody)
|
||||
req.RemoteAddr = remoteAddr
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
resp := w.Result()
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// one request a second is allowed, so the first of a burst passes and the rest are refused
|
||||
assert.Equal(t, http.StatusOK, call("1.2.3.4:1000"), "first request within the allowance")
|
||||
refused := 0
|
||||
for range 5 {
|
||||
if call("1.2.3.4:1000") == http.StatusTooManyRequests {
|
||||
refused++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 5, refused, "burst past the allowance is refused")
|
||||
|
||||
// a different client has its own bucket and is unaffected
|
||||
assert.Equal(t, http.StatusOK, call("5.6.7.8:1000"), "limit is per client, not global")
|
||||
}
|
||||
|
||||
func TestRealIPMiddleware(t *testing.T) {
|
||||
// call runs mw with the given peer and (optional) X-Real-IP header and returns what the
|
||||
// downstream handler observes; state is per-call, so subtests don't share closure locals.
|
||||
@@ -238,11 +271,14 @@ func TestRest_securityHeaders(t *testing.T) {
|
||||
client := http.Client{}
|
||||
resp, err := client.Get(ts.URL + "/web/index.html")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "img-src *;")
|
||||
assert.Equal(t, "nosniff", resp.Header.Get("X-Content-Type-Options"))
|
||||
assert.Equal(t, "strict-origin-when-cross-origin", resp.Header.Get("Referrer-Policy"))
|
||||
// httptest.Server.Close waits on connections still in use, and a deferred close does not run
|
||||
// until the test ends, so the body has to be released before the server is torn down here
|
||||
require.NoError(t, resp.Body.Close())
|
||||
client.CloseIdleConnections()
|
||||
teardown()
|
||||
|
||||
// check CSP with proxy enabled
|
||||
|
||||
@@ -4,11 +4,13 @@ import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -18,6 +20,7 @@ import (
|
||||
|
||||
"github.com/umputun/remark42/backend/app/migrator"
|
||||
"github.com/umputun/remark42/backend/app/rest"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
)
|
||||
|
||||
// Migrator rest with import and export controllers
|
||||
@@ -151,7 +154,8 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&buf)
|
||||
if _, err := m.NativeExporter.Export(gzWriter, siteID); err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
|
||||
code, errCode := exportErrStatus(err)
|
||||
rest.SendErrorJSON(w, r, code, err, "export failed", errCode)
|
||||
return
|
||||
}
|
||||
if err := gzWriter.Close(); err != nil {
|
||||
@@ -171,10 +175,23 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// stream mode - write directly to response
|
||||
if _, err := m.NativeExporter.Export(w, siteID); err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
|
||||
code, errCode := exportErrStatus(err)
|
||||
rest.SendErrorJSON(w, r, code, err, "export failed", errCode)
|
||||
}
|
||||
}
|
||||
|
||||
// exportErrStatus maps an export failure to an HTTP status and error code: an unknown
|
||||
// site is a client error (400), anything else is treated as internal (500).
|
||||
// The bolt store returns the engine.ErrSiteNotFound sentinel; the rpc store loses typed
|
||||
// errors over jrpc, so the "not found" message is matched as a fallback (export only ever
|
||||
// hits a site-level lookup, so a "not found" here can only mean the site).
|
||||
func exportErrStatus(err error) (status, errCode int) {
|
||||
if errors.Is(err, engine.ErrSiteNotFound) || strings.Contains(err.Error(), "not found") {
|
||||
return http.StatusBadRequest, rest.ErrSiteNotFound
|
||||
}
|
||||
return http.StatusInternalServerError, rest.ErrInternal
|
||||
}
|
||||
|
||||
// POST /remap?site=site-id
|
||||
// remap urls in comments based on given rules (oldUrl newUrl)
|
||||
func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestMigrator_Import(t *testing.T) {
|
||||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||||
require.NoError(t, err)
|
||||
@@ -125,7 +125,7 @@ func TestMigrator_ImportFromWP(t *testing.T) {
|
||||
|
||||
r := strings.NewReader(strings.ReplaceAll(xmlTestWP, "'", "`"))
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=wordpress", r)
|
||||
assert.NoError(t, err)
|
||||
@@ -170,7 +170,7 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
|
||||
"email":"somegreatmail@gmail.com","name":"User5276","link":"https://example.com/profile/257","photo":"https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
|
||||
"provider":"sso:example.com","joinDate":"2021-03-19T19:27:25.954285Z","isModerator":false}]}`)
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
|
||||
assert.NoError(t, err)
|
||||
@@ -211,7 +211,7 @@ func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
|
||||
r, err := os.Open("testdata/commento.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
|
||||
assert.NoError(t, err)
|
||||
@@ -258,7 +258,7 @@ func TestMigrator_ImportRejected(t *testing.T) {
|
||||
"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"remark42","url":"https://radio-t.com/blah2"},"score":0,
|
||||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native&secret=XYZ", r)
|
||||
assert.NoError(t, err)
|
||||
@@ -280,10 +280,14 @@ func TestMigrator_ImportDouble(t *testing.T) {
|
||||
for i := range 50 {
|
||||
recs = append(recs, fmt.Sprintf(tmpl, i))
|
||||
}
|
||||
r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
// each request needs its own reader. client.Do returns once the response headers are in, which
|
||||
// for an accepted import is before the transport's writeLoop has finished copying the body, so
|
||||
// handing the same strings.Reader to the second NewRequest races that copy: NewRequest reads
|
||||
// Len() to set ContentLength while WriteTo is still advancing it
|
||||
body := `{"version":1}` + strings.Join(recs, "\n")
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
assert.NoError(t, err)
|
||||
@@ -294,7 +298,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
|
||||
|
||||
client = &http.Client{Timeout: 5 * time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||||
req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
assert.NoError(t, err)
|
||||
@@ -380,7 +384,7 @@ func TestMigrator_Export(t *testing.T) {
|
||||
"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
|
||||
|
||||
// import comments first
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=native", r)
|
||||
require.NoError(t, err)
|
||||
@@ -391,15 +395,31 @@ func TestMigrator_Export(t *testing.T) {
|
||||
require.Equal(t, http.StatusAccepted, resp.StatusCode)
|
||||
waitForMigrationCompletion(t, ts)
|
||||
|
||||
// export wrong site, should result in error
|
||||
// export unknown site is a client error, not internal
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=test", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
resp, err = client.Do(req)
|
||||
require.NoError(t, err)
|
||||
errBody, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
|
||||
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||||
require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
|
||||
assert.Contains(t, string(errBody), `"code":6`) // rest.ErrSiteNotFound, not ErrInternal
|
||||
assert.Contains(t, string(errBody), `not found`) // error detail names the missing site
|
||||
|
||||
// unknown site in stream mode is also a client error
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=test", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
req.SetBasicAuth("admin", "password")
|
||||
resp, err = client.Do(req)
|
||||
require.NoError(t, err)
|
||||
errBody, err = io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||||
assert.Contains(t, string(errBody), `"code":6`)
|
||||
|
||||
// check file mode
|
||||
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=remark42", http.NoBody)
|
||||
@@ -557,7 +577,7 @@ func TestMigrator_RemapReject(t *testing.T) {
|
||||
defer teardown()
|
||||
|
||||
// without admin credentials
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
rules := strings.NewReader(`https://remark42.com/* https://www.remark42.com/*`)
|
||||
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/remap?site=remark42", rules)
|
||||
|
||||
@@ -3,7 +3,6 @@ package api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -27,6 +26,7 @@ import (
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/image"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark42/backend/app/webassets"
|
||||
)
|
||||
|
||||
// Rest is a rest access server
|
||||
@@ -45,7 +45,7 @@ type Rest struct {
|
||||
|
||||
AnonVote bool
|
||||
WebRoot string
|
||||
WebFS embed.FS
|
||||
WebFS fs.FS
|
||||
RemarkURL string
|
||||
ReadOnlyAge int
|
||||
SharedSecret string
|
||||
@@ -385,8 +385,16 @@ func (s *Rest) routes() http.Handler {
|
||||
rroot.HandleFunc("POST /email/unsubscribe.html", s.privRest.emailUnsubscribeCtrl)
|
||||
})
|
||||
|
||||
// file server for static content from s.WebRoot on path /web
|
||||
addFileServer(router, s.WebFS, s.WebRoot, s.Version)
|
||||
// file server for /web: the frontend build first, then the assets embedded in the binary.
|
||||
// the build is embedded under web/ by app/cmd, so that prefix is stripped here. fs.Sub only
|
||||
// fails for an fs.SubFS that refuses, and a nil result would panic on the first request, so
|
||||
// serve nothing from the frontend rather than serving it at the wrong paths
|
||||
embeddedFrontend, err := fs.Sub(s.WebFS, "web")
|
||||
if err != nil {
|
||||
log.Printf("[WARN] no embedded frontend, serving built-in assets only: %v", err)
|
||||
embeddedFrontend = emptyFS{}
|
||||
}
|
||||
addFileServer(router, embeddedFrontend, s.WebRoot, s.Version, s.RemarkURL)
|
||||
return router
|
||||
}
|
||||
|
||||
@@ -499,25 +507,35 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
R.RenderJSON(w, cnf)
|
||||
}
|
||||
|
||||
// serves static files from the webRoot directory or files embedded into the compiled binary if that directory is absent
|
||||
func addFileServer(r *routegroup.Bundle, embedFS embed.FS, webRoot, version string) {
|
||||
var webFS http.Handler
|
||||
// serves /web from the frontend build, falling back to the assets embedded in the binary for
|
||||
// names the build does not produce. the frontend build is read from webRoot on disk, or from the
|
||||
// copy embedded at app/cmd/web when that directory is absent.
|
||||
func addFileServer(r *routegroup.Bundle, embeddedFrontend fs.FS, webRoot, version, remarkURL string) {
|
||||
frontendFS := embeddedFrontend
|
||||
|
||||
if _, err := os.Stat(webRoot); err == nil {
|
||||
log.Printf("[INFO] run file server from %s from the disk", webRoot)
|
||||
webFS = http.FileServer(http.Dir(webRoot))
|
||||
frontendFS = os.DirFS(webRoot)
|
||||
} else {
|
||||
log.Printf("[INFO] run file server, embedded")
|
||||
var contentFS, _ = fs.Sub(embedFS, "web")
|
||||
webFS = http.FileServer(http.FS(contentFS))
|
||||
}
|
||||
|
||||
webFS = http.StripPrefix("/web", webFS)
|
||||
// wrapped rather than substituted once at startup: the disk root can change under a running
|
||||
// server, and the docker image has already substituted its copy, where this is a no-op
|
||||
sources := templatedFS{
|
||||
fs: webFiles{frontend: frontendFS, embedded: webassets.FS},
|
||||
remarkURL: remarkURL,
|
||||
}
|
||||
webFS := http.StripPrefix("/web", http.FileServer(http.FS(sources)))
|
||||
r.HandleFunc("GET /web", http.RedirectHandler("/web/", http.StatusMovedPermanently).ServeHTTP)
|
||||
|
||||
r.With(rateLimiter(20),
|
||||
R.Timeout(10*time.Second),
|
||||
cacheControl(time.Hour, version),
|
||||
// the served body now depends on remarkURL, so it has to be part of the validator. Without
|
||||
// it an operator who corrects a wrong REMARK_URL and restarts the same binary keeps getting
|
||||
// 304 on revalidation, and the client keeps a bundle addressed to the old host for good,
|
||||
// since no-cache means it revalidates rather than aging out
|
||||
cacheControl(time.Hour, version+":"+remarkURL),
|
||||
).HandleFunc("GET /web/", func(w http.ResponseWriter, r *http.Request) {
|
||||
// don't show dirs, just serve files
|
||||
if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != ("/web/") {
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
log "github.com/go-pkgz/lgr"
|
||||
R "github.com/go-pkgz/rest"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/notify"
|
||||
"github.com/umputun/remark42/backend/app/rest"
|
||||
@@ -663,10 +662,8 @@ func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
return e
|
||||
}
|
||||
|
||||
var merr error
|
||||
merr = multierror.Append(merr, write([]byte(`{"info": `))) // send user prefix
|
||||
merr = multierror.Append(merr, write(userB)) // send user info
|
||||
merr = multierror.Append(merr, write([]byte(`, "comments":`))) // send comments prefix
|
||||
// send user prefix, user info and comments prefix
|
||||
errs := []error{write([]byte(`{"info": `)), write(userB), write([]byte(`, "comments":`))}
|
||||
|
||||
// get comments in 100 in each paginated request
|
||||
for i := range 100 {
|
||||
@@ -681,15 +678,15 @@ func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
merr = multierror.Append(merr, write(b))
|
||||
errs = append(errs, write(b))
|
||||
if len(comments) != 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
merr = multierror.Append(merr, write([]byte(`}`)))
|
||||
if merr.(*multierror.Error).ErrorOrNil() != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info", rest.ErrInternal)
|
||||
errs = append(errs, write([]byte(`}`)))
|
||||
if err := errors.Join(errs...); err != nil {
|
||||
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't write user info", rest.ErrInternal)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ func TestRest_CreateAndPreviewWithImage(t *testing.T) {
|
||||
|
||||
assert.Equal(t, false, pngRead, "original image is not yet accessed by server")
|
||||
// retrieve the image from the cache
|
||||
imgURL := strings.Split(strings.Split(string(b), "src=\"")[1], "\"")[0]
|
||||
imgURL, _, _ := strings.Cut(strings.Split(string(b), "src=\"")[1], "\"")
|
||||
// replace srv.RemarkURL with ts.URL
|
||||
imgURL = strings.ReplaceAll(imgURL, srv.RemarkURL, ts.URL)
|
||||
resp, err = http.Get(imgURL)
|
||||
@@ -432,6 +432,7 @@ func TestRest_Update(t *testing.T) {
|
||||
strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
|
||||
assert.NoError(t, err)
|
||||
req.Header.Add("X-JWT", devToken)
|
||||
beforeUpdate := time.Now()
|
||||
b, err := client.Do(req)
|
||||
assert.NoError(t, err)
|
||||
body, err := io.ReadAll(b.Body)
|
||||
@@ -447,7 +448,7 @@ func TestRest_Update(t *testing.T) {
|
||||
assert.Equal(t, "<p>updated text</p>\n", c2.Text)
|
||||
assert.Equal(t, "updated text", c2.Orig)
|
||||
assert.Equal(t, "my edit", c2.Edit.Summary)
|
||||
assert.True(t, time.Since(c2.Edit.Timestamp) < 1*time.Second)
|
||||
assert.WithinRange(t, c2.Edit.Timestamp, beforeUpdate, time.Now(), "edit stamped during the update")
|
||||
|
||||
// read updated comment
|
||||
res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, id))
|
||||
@@ -596,7 +597,7 @@ func TestRest_DeleteChildThenParent(t *testing.T) {
|
||||
fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah1", ts.URL, idC2), http.NoBody)
|
||||
require.NoError(t, err)
|
||||
requireAdminOnly(t, req)
|
||||
resp, err = sendReq(t, req, adminUmputunToken)
|
||||
resp, err = sendReq(req, adminUmputunToken)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
@@ -784,7 +785,7 @@ func TestRest_Vote(t *testing.T) {
|
||||
req, err := http.NewRequest("GET",
|
||||
fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), http.NoBody)
|
||||
assert.NoError(t, err)
|
||||
resp, err := sendReq(t, req, adminUmputunToken)
|
||||
resp, err := sendReq(req, adminUmputunToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
cr = store.Comment{}
|
||||
@@ -974,9 +975,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
parentComment := store.Comment{}
|
||||
require.NoError(t, json.Unmarshal(body, &parentComment))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 1, len(mockDestination.Get()))
|
||||
waitForCount(t, 1, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[0].Emails)
|
||||
|
||||
// create child comment from another user, email notification only to admin expected
|
||||
@@ -994,9 +993,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 2, len(mockDestination.Get()))
|
||||
waitForCount(t, 2, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[1].Emails)
|
||||
|
||||
// send confirmation token for email
|
||||
@@ -1013,9 +1010,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 1, len(mockDestination.GetVerify()))
|
||||
waitForCount(t, 1, func() int { return len(mockDestination.GetVerify()) })
|
||||
assert.Equal(t, "good@example.com", mockDestination.GetVerify()[0].Email)
|
||||
verificationToken := mockDestination.GetVerify()[0].Token
|
||||
|
||||
@@ -1087,9 +1082,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 3, len(mockDestination.Get()))
|
||||
waitForCount(t, 3, func() int { return len(mockDestination.Get()) })
|
||||
assert.Equal(t, []string{"good@example.com"}, mockDestination.Get()[2].Emails)
|
||||
|
||||
// delete user's email
|
||||
@@ -1117,9 +1110,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 4, len(mockDestination.Get()))
|
||||
waitForCountSettled(t, 4, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[3].Emails)
|
||||
|
||||
// confirm email via subscribe call with query params, old behavior, email notification is expected
|
||||
@@ -1136,9 +1127,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 2, len(mockDestination.GetVerify()), "verification email was sent")
|
||||
waitForCount(t, 2, func() int { return len(mockDestination.GetVerify()) }, "verification email was sent")
|
||||
|
||||
// get email user information to verify there is no subscription yet
|
||||
req, err = http.NewRequest(
|
||||
@@ -1173,9 +1162,7 @@ func TestRest_EmailNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 2, len(mockDestination.GetVerify()), "no new verification email was sent")
|
||||
waitForCountSettled(t, 2, func() int { return len(mockDestination.GetVerify()) }, "no new verification email was sent")
|
||||
|
||||
// get email user information to verify the subscription happened without the confirmation call
|
||||
req, err = http.NewRequest(
|
||||
@@ -1224,9 +1211,7 @@ func TestRest_TelegramNotification(t *testing.T) {
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
parentComment := store.Comment{}
|
||||
require.NoError(t, json.Unmarshal(body, &parentComment))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 1, len(mockDestination.Get()))
|
||||
waitForCount(t, 1, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[0].Telegrams)
|
||||
|
||||
// create child comment from another user, telegram notification only to admin expected
|
||||
@@ -1244,9 +1229,7 @@ func TestRest_TelegramNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 2, len(mockDestination.Get()))
|
||||
waitForCount(t, 2, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[1].Telegrams)
|
||||
|
||||
// subscribe to telegram while the telegram destination is absent
|
||||
@@ -1357,9 +1340,7 @@ func TestRest_TelegramNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 3, len(mockDestination.Get()))
|
||||
waitForCount(t, 3, func() int { return len(mockDestination.Get()) })
|
||||
assert.Equal(t, []string{"good_telegram"}, mockDestination.Get()[2].Telegrams)
|
||||
|
||||
// delete user's telegram
|
||||
@@ -1387,9 +1368,7 @@ func TestRest_TelegramNotification(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
|
||||
// wait for mock notification Submit to kick off
|
||||
time.Sleep(time.Millisecond * 30)
|
||||
require.Equal(t, 4, len(mockDestination.Get()))
|
||||
waitForCountSettled(t, 4, func() int { return len(mockDestination.Get()) })
|
||||
assert.Empty(t, mockDestination.Get()[3].Telegrams)
|
||||
}
|
||||
|
||||
@@ -1412,7 +1391,7 @@ func TestRest_UserAllData(t *testing.T) {
|
||||
_, err = srv.DataService.Create(c3)
|
||||
require.NoError(t, err)
|
||||
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=remark42", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
@@ -1465,7 +1444,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
|
||||
_, err := srv.DataService.Create(c)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
client := &http.Client{Timeout: waitTimeout}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=remark42", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
@@ -1608,7 +1587,9 @@ func TestRest_CreateWithPictures(t *testing.T) {
|
||||
Staging: "/tmp/remark42/images.staging",
|
||||
Location: "/tmp/remark42/images",
|
||||
}, image.ServiceParams{
|
||||
EditDuration: 100 * time.Millisecond,
|
||||
// the "not moved yet" checks below run right after the comment POST returns, so the
|
||||
// commit window has to be wide enough that a stalled runner cannot close it first
|
||||
EditDuration: 3 * time.Second,
|
||||
MaxSize: 2000,
|
||||
ImageAPI: svc.RemarkURL + "/api/v1/picture/",
|
||||
ProxyAPI: svc.RemarkURL + "/api/v1/img",
|
||||
@@ -1671,11 +1652,12 @@ func TestRest_CreateWithPictures(t *testing.T) {
|
||||
assert.Error(t, err, "picture %d not moved from staging yet", i)
|
||||
}
|
||||
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
|
||||
// the commit runs once EditDuration expires
|
||||
for i := range ids {
|
||||
_, err = os.Stat("/tmp/remark42/images/" + ids[i])
|
||||
assert.NoError(t, err, "picture %d moved from staging and available in permanent location", i)
|
||||
require.Eventually(t, func() bool {
|
||||
_, e := os.Stat("/tmp/remark42/images/" + ids[i])
|
||||
return e == nil
|
||||
}, waitTimeout, pollInterval, "picture %d moved from staging and available in permanent location", i)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image/png"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -381,11 +382,12 @@ func TestRest_Last(t *testing.T) {
|
||||
c2 := store.Comment{Text: "test test #2", ParentID: "p1",
|
||||
Locator: store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah2"}}
|
||||
|
||||
// add 3 comments
|
||||
// add 3 comments, with the clock pushed past a millisecond boundary in between so the two
|
||||
// "since" values below are distinct
|
||||
ts1 := time.Now().UnixNano() / 1000000
|
||||
addComment(t, c1, ts)
|
||||
id1 := addComment(t, c1, ts)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
waitPastMillisecond(time.Now())
|
||||
ts2 := time.Now().UnixNano() / 1000000
|
||||
id2 := addComment(t, c2, ts)
|
||||
|
||||
@@ -539,9 +541,17 @@ func TestRest_FindUserComments_CWE_918(t *testing.T) {
|
||||
assert.Equal(t, arbitraryServer.URL, resp.Comments[0].Locator.URL, "arbitrary URL provided by the request")
|
||||
}
|
||||
|
||||
// waitPastMillisecond blocks until the wall clock moves past ts's millisecond, so whatever is
|
||||
// created next gets a distinct value for the millisecond-precision "since" filter
|
||||
func waitPastMillisecond(ts time.Time) {
|
||||
next := ts.Truncate(time.Millisecond).Add(time.Millisecond)
|
||||
time.Sleep(time.Until(next) + time.Microsecond) // a non-positive duration returns at once
|
||||
}
|
||||
|
||||
func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
|
||||
// test that comment counting is consistent between tree and plain formats
|
||||
ts, srv, teardown := startupT(t)
|
||||
// the open-route limit is lifted so the subtests below can run back to back
|
||||
ts, srv, teardown := startupT(t, func(srv *Rest) { srv.openRouteLimiter = 100000 })
|
||||
defer teardown()
|
||||
|
||||
commentLocator := store.Locator{URL: "test-url", SiteID: "remark42"}
|
||||
@@ -567,55 +577,55 @@ func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
|
||||
}
|
||||
|
||||
// adding initial comments (8 to test-url and 1 to another-url) and voting, and delete two of comments to the first post.
|
||||
// with sleep so that at least few millisecond pass between each comment
|
||||
// and later we would be able to use that in "since" filter with millisecond precision
|
||||
// each comment waits for the clock to pass the previous one's millisecond so the "since"
|
||||
// filter, which has millisecond precision, can tell them apart
|
||||
ids := make([]string, 9)
|
||||
timestamps := make([]time.Time, 9)
|
||||
c1 := store.Comment{Text: "top-level comment 1", Locator: commentLocator}
|
||||
ids[0], timestamps[0] = addCommentGetCreatedTime(t, c1, ts)
|
||||
// #3 by score
|
||||
setScore(commentLocator, ids[0], 1)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[0])
|
||||
|
||||
c2 := store.Comment{Text: "top-level comment 2", Locator: commentLocator}
|
||||
ids[1], timestamps[1] = addCommentGetCreatedTime(t, c2, ts)
|
||||
// #2 by score
|
||||
setScore(commentLocator, ids[1], 2)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[1])
|
||||
|
||||
c3 := store.Comment{Text: "second-level comment 1", ParentID: ids[0], Locator: commentLocator}
|
||||
ids[2], timestamps[2] = addCommentGetCreatedTime(t, c3, ts)
|
||||
// #1 by score
|
||||
setScore(commentLocator, ids[2], 10)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[2])
|
||||
|
||||
c4 := store.Comment{Text: "third-level comment 1", ParentID: ids[2], Locator: commentLocator}
|
||||
ids[3], timestamps[3] = addCommentGetCreatedTime(t, c4, ts)
|
||||
// #5 by score, #1 by controversy
|
||||
setScore(commentLocator, ids[3], 4)
|
||||
setScore(commentLocator, ids[3], -4)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[3])
|
||||
|
||||
c5 := store.Comment{Text: "second-level comment 2", ParentID: ids[1], Locator: commentLocator}
|
||||
ids[4], timestamps[4] = addCommentGetCreatedTime(t, c5, ts)
|
||||
// #5 by score, #2 by controversy
|
||||
setScore(commentLocator, ids[4], 2)
|
||||
setScore(commentLocator, ids[4], -3)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[4])
|
||||
|
||||
c6 := store.Comment{Text: "deleted third-level comment 2", ParentID: ids[4], Locator: commentLocator}
|
||||
ids[5], timestamps[5] = addCommentGetCreatedTime(t, c6, ts)
|
||||
// deleted later so not visible in site-wide requests
|
||||
setScore(commentLocator, ids[5], 10)
|
||||
setScore(commentLocator, ids[5], -10)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[5])
|
||||
|
||||
c7 := store.Comment{Text: "top-level comment 3", Locator: commentLocator}
|
||||
ids[6], timestamps[6] = addCommentGetCreatedTime(t, c7, ts)
|
||||
// #6 by score, #4 by controversy
|
||||
setScore(commentLocator, ids[6], -3)
|
||||
setScore(commentLocator, ids[6], 1)
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
waitPastMillisecond(timestamps[6])
|
||||
|
||||
c8 := store.Comment{Text: "deleted second-level comment 3", ParentID: ids[6], Locator: commentLocator}
|
||||
ids[7], timestamps[7] = addCommentGetCreatedTime(t, c8, ts)
|
||||
@@ -733,16 +743,16 @@ func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
|
||||
{"format=tree&limit=bad", `{"code":1,"details":"bad limit value","error":"strconv.Atoi: parsing \"bad\": invalid syntax"}`},
|
||||
{"format=tree&offset_id=bad", `{"code":1,"details":"bad offset_id value","error":"invalid UUID length: 3"}`},
|
||||
{"format=tree&limit=2", `"info":{"count":7,"count_left":4,"last_comment":"` + ids[0]},
|
||||
{"format=tree&limit=6", `"info":{"count":7,"count_left":2,"last_comment":"` + ids[1]},
|
||||
{"format=tree&limit=7", `"info":{"count":7,"count_left":1,"last_comment":"` + ids[6]},
|
||||
{"format=tree&limit=6", `"info":{"count":7,"count_left":1,"last_comment":"` + ids[6]},
|
||||
{"format=tree&limit=7", `"info":{"count":7,"count_left":0,"last_comment":"` + ids[8]},
|
||||
{"format=tree&url=test-url&limit=2", `"info":{"url":"test-url","count":6,"count_left":3,"last_comment":"` + ids[0]},
|
||||
{"format=tree&url=test-url&limit=6", `"info":{"url":"test-url","count":6,"count_left":1,"last_comment":"` + ids[1]},
|
||||
{"format=tree&url=test-url&limit=6", `"info":{"url":"test-url","count":6,"count_left":0,"last_comment":"` + ids[6]},
|
||||
{"format=tree&url=test-url&limit=7", `"info":{"url":"test-url","count":6,"count_left":0,"last_comment":"` + ids[6]},
|
||||
// start after first top-level comment
|
||||
{fmt.Sprintf("format=tree&limit=2&offset_id=%s", ids[0]), `"info":{"count":7,"count_left":2,"last_comment":"` + ids[1]},
|
||||
{fmt.Sprintf("format=tree&url=test-url&limit=2&offset_id=%s", ids[0]), `"info":{"url":"test-url","count":6,"count_left":1,"last_comment":"` + ids[1]},
|
||||
// start after second top-level comment
|
||||
{fmt.Sprintf("format=tree&limit=2&offset_id=%s", ids[1]), `"info":{"count":7,"count_left":1,"last_comment":"` + ids[6]},
|
||||
{fmt.Sprintf("format=tree&limit=2&offset_id=%s", ids[1]), `"info":{"count":7,"count_left":0,"last_comment":"` + ids[8]},
|
||||
{fmt.Sprintf("format=tree&url=test-url&limit=2&offset_id=%s", ids[1]), `"info":{"url":"test-url","count":6,"count_left":0,"last_comment":"` + ids[6]},
|
||||
// start after third top-level comment, so expect comment to post 2, or no comments on post 1 if "url" is set
|
||||
{fmt.Sprintf("format=tree&limit=1&offset_id=%s", ids[6]), `"info":{"count":7,"count_left":0,"last_comment":"` + ids[8]},
|
||||
@@ -771,15 +781,17 @@ func TestPublic_FindCommentsCtrl_ConsistentCount(t *testing.T) {
|
||||
t.Run(tc.params, func(t *testing.T) {
|
||||
url := fmt.Sprintf(ts.URL+"/api/v1/find?site=remark42&%s", tc.params)
|
||||
body, code := get(t, url)
|
||||
// bad-request cases are identified by their error response body rather than
|
||||
// a "=bad" substring of the params: comment IDs are random UUIDs and one
|
||||
// starting with "bad" (e.g. offset_id=bad49e60-...) would otherwise be
|
||||
// misread as a bad request, making this test flaky.
|
||||
expectedStatus := http.StatusOK
|
||||
if strings.Contains(tc.params, "=bad") {
|
||||
if strings.Contains(tc.expectedBody, `"error":`) {
|
||||
expectedStatus = http.StatusBadRequest
|
||||
}
|
||||
assert.Equal(t, expectedStatus, code)
|
||||
assert.Contains(t, body, tc.expectedBody)
|
||||
t.Log(body)
|
||||
// prevent hit limiter from engaging
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -975,13 +987,26 @@ func TestRest_QR(t *testing.T) {
|
||||
assert.Equal(t, "image/png", r.Header.Get("Content-Type"))
|
||||
assert.Equal(t, http.StatusOK, r.StatusCode)
|
||||
|
||||
// compare the image
|
||||
// compare the decoded image rather than the encoded bytes: the pixels are what the endpoint
|
||||
// promises, while the byte stream is whatever the toolchain's png encoder produces, and
|
||||
// pinning that fails on a go release that changes it
|
||||
fh, err := os.Open("testdata/qr_test.png")
|
||||
defer func() { assert.NoError(t, fh.Close()) }()
|
||||
assert.NoError(t, err)
|
||||
img, err := io.ReadAll(fh)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, img, bdy)
|
||||
require.NoError(t, err)
|
||||
|
||||
want, err := png.Decode(fh)
|
||||
require.NoError(t, err)
|
||||
got, err := png.Decode(bytes.NewReader(bdy))
|
||||
require.NoError(t, err, "the endpoint did not return a decodable png")
|
||||
|
||||
require.Equal(t, want.Bounds(), got.Bounds(), "the qr code is not the size it used to be")
|
||||
for y := want.Bounds().Min.Y; y < want.Bounds().Max.Y; y++ {
|
||||
for x := want.Bounds().Min.X; x < want.Bounds().Max.X; x++ {
|
||||
if want.At(x, y) != got.At(x, y) {
|
||||
t.Fatalf("the qr code differs at %d,%d: want %v, got %v", x, y, want.At(x, y), got.At(x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRest_Info(t *testing.T) {
|
||||
|
||||
@@ -4,16 +4,19 @@ import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"io/fs"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/go-pkgz/auth/v2"
|
||||
@@ -22,6 +25,7 @@ import (
|
||||
"github.com/go-pkgz/auth/v2/token"
|
||||
cache "github.com/go-pkgz/lcw/v2"
|
||||
R "github.com/go-pkgz/rest"
|
||||
"github.com/go-pkgz/routegroup"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
@@ -36,6 +40,7 @@ import (
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/image"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark42/backend/app/webassets"
|
||||
)
|
||||
|
||||
// To generate a token, enter one of the tokens here into https://jwt.io, change the secret to one you're using in your test
|
||||
@@ -118,6 +123,182 @@ func TestRest_FileServerStaticAssets(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestRest_FileServerBackendAssets covers the assets embedded in the binary and the rule that a
|
||||
// name the frontend build provides is served from there instead. WebRoot is a fresh empty
|
||||
// directory so the frontend side is known, rather than the shared temp dir startupT defaults to.
|
||||
func TestRest_FileServerBackendAssets(t *testing.T) {
|
||||
ts, srv, teardown := startupT(t, func(srv *Rest) { srv.WebRoot = t.TempDir() })
|
||||
defer teardown()
|
||||
|
||||
t.Run("serves every embedded asset byte for byte", func(t *testing.T) {
|
||||
for _, name := range []string{"privacy.html", "markdown-help.html", "400x400.jpeg"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
want, err := fs.ReadFile(webassets.FS, name)
|
||||
require.NoError(t, err)
|
||||
|
||||
body, code := get(t, ts.URL+"/web/"+name)
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, string(want), body, "the bytes must come from the embedded assets")
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("serves the image with its own content type", func(t *testing.T) {
|
||||
resp, err := http.Get(ts.URL + "/web/400x400.jpeg")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
|
||||
})
|
||||
|
||||
t.Run("head is served", func(t *testing.T) {
|
||||
resp, err := http.Head(ts.URL + "/web/privacy.html")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("frontend output wins over the embedded copy", func(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(srv.WebRoot+"/privacy.html", []byte("operator's own policy"), 0o600))
|
||||
t.Cleanup(func() { _ = os.Remove(srv.WebRoot + "/privacy.html") })
|
||||
|
||||
body, code := get(t, ts.URL+"/web/privacy.html")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, "operator's own policy", body)
|
||||
})
|
||||
|
||||
t.Run("traversal out of the asset root is refused", func(t *testing.T) {
|
||||
for _, p := range []string{"/web/../../etc/passwd", "/web/..%2f..%2fetc%2fpasswd", "/web/%2e%2e/%2e%2e/etc/passwd"} {
|
||||
t.Run(p, func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+p)
|
||||
assert.NotContains(t, body, "root:", "must never serve a file outside the served roots")
|
||||
assert.NotEqual(t, http.StatusInternalServerError, code, "a rejected name must not surface as 500")
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing in both still returns 404", func(t *testing.T) {
|
||||
_, code := get(t, ts.URL+"/web/neither-source-has-this.html")
|
||||
assert.Equal(t, http.StatusNotFound, code)
|
||||
})
|
||||
}
|
||||
|
||||
// TestRest_FileServerEmbeddedFrontend covers the branch taken when no web root exists on disk,
|
||||
// which is how the released binary runs. The frontend stands in for the copy embedded at
|
||||
// app/cmd/web, so a name it provides and a name only the assets provide are both exercised.
|
||||
func TestRest_FileServerEmbeddedFrontend(t *testing.T) {
|
||||
frontend := fstest.MapFS{"index.html": {Data: []byte("embedded frontend index")}}
|
||||
router := routegroup.New(http.NewServeMux())
|
||||
addFileServer(router, frontend, filepath.Join(t.TempDir(), "absent"), "test-version", "https://remark.example.com")
|
||||
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
t.Run("serves the embedded frontend", func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/index.html")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, "embedded frontend index", body)
|
||||
})
|
||||
|
||||
for _, name := range []string{"privacy.html", "markdown-help.html", "400x400.jpeg"} {
|
||||
t.Run("falls back to "+name, func(t *testing.T) {
|
||||
want, err := fs.ReadFile(webassets.FS, name)
|
||||
require.NoError(t, err)
|
||||
|
||||
body, code := get(t, ts.URL+"/web/"+name)
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, string(want), body)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a name neither source has is missing", func(t *testing.T) {
|
||||
_, code := get(t, ts.URL+"/web/nothing-here.html")
|
||||
assert.Equal(t, http.StatusNotFound, code)
|
||||
})
|
||||
|
||||
t.Run("a name the operating system rejects is missing, not an error", func(t *testing.T) {
|
||||
_, code := get(t, ts.URL+"/web/a%00b.html")
|
||||
assert.Equal(t, http.StatusNotFound, code)
|
||||
})
|
||||
}
|
||||
|
||||
// TestRest_FileServerRoutesEmbedded drives the whole router the released binary runs: no web root
|
||||
// on disk, and the frontend read from WebFS. It is what pins the web/ prefix routes() strips, which
|
||||
// a test calling addFileServer directly cannot see.
|
||||
func TestRest_FileServerRoutesEmbedded(t *testing.T) {
|
||||
frontend := fstest.MapFS{
|
||||
"web/index.html": {Data: []byte("embedded index")},
|
||||
"web/iframe.html": {Data: []byte("embedded iframe")},
|
||||
"web/remark.mjs": {Data: []byte("embedded bundle")},
|
||||
}
|
||||
ts, _, teardown := startupT(t, func(srv *Rest) {
|
||||
srv.WebRoot = filepath.Join(t.TempDir(), "absent")
|
||||
srv.WebFS = frontend
|
||||
})
|
||||
defer teardown()
|
||||
|
||||
t.Run("serves the frontend from under the web prefix", func(t *testing.T) {
|
||||
for name, want := range map[string]string{
|
||||
"index.html": "embedded index",
|
||||
"iframe.html": "embedded iframe",
|
||||
"remark.mjs": "embedded bundle",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/"+name)
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, want, body)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the prefix is stripped rather than exposed", func(t *testing.T) {
|
||||
_, code := get(t, ts.URL+"/web/web/index.html")
|
||||
assert.Equal(t, http.StatusNotFound, code, "the web/ prefix must not be reachable as a path")
|
||||
})
|
||||
|
||||
t.Run("the embedded assets still answer alongside it", func(t *testing.T) {
|
||||
want, err := fs.ReadFile(webassets.FS, "privacy.html")
|
||||
require.NoError(t, err)
|
||||
|
||||
body, code := get(t, ts.URL+"/web/privacy.html")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, string(want), body)
|
||||
})
|
||||
}
|
||||
|
||||
// refusingSubFS is an fs.FS whose Sub refuses, which is the only way fs.Sub returns a nil
|
||||
// filesystem. routes() has to survive it, since a nil frontend would panic on the first request.
|
||||
type refusingSubFS struct{}
|
||||
|
||||
func (refusingSubFS) Open(name string) (fs.File, error) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
func (refusingSubFS) Sub(string) (fs.FS, error) { return nil, errors.New("refused") }
|
||||
|
||||
// TestRest_FileServerFrontendSourceRefused covers the branch where the frontend source cannot be
|
||||
// sub-rooted: /web must keep serving the embedded assets rather than panicking.
|
||||
func TestRest_FileServerFrontendSourceRefused(t *testing.T) {
|
||||
ts, _, teardown := startupT(t, func(srv *Rest) {
|
||||
srv.WebRoot = filepath.Join(t.TempDir(), "absent")
|
||||
srv.WebFS = refusingSubFS{}
|
||||
})
|
||||
defer teardown()
|
||||
|
||||
t.Run("the embedded assets still serve", func(t *testing.T) {
|
||||
want, err := fs.ReadFile(webassets.FS, "privacy.html")
|
||||
require.NoError(t, err)
|
||||
|
||||
body, code := get(t, ts.URL+"/web/privacy.html")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, string(want), body)
|
||||
})
|
||||
|
||||
t.Run("a frontend name is missing rather than fatal", func(t *testing.T) {
|
||||
_, code := get(t, ts.URL+"/web/iframe.html")
|
||||
assert.Equal(t, http.StatusNotFound, code)
|
||||
})
|
||||
}
|
||||
|
||||
// TestRest_RejectHeadOnDestructiveGET verifies that HEAD is blocked on the state-mutating
|
||||
// GET routes (which stdlib http.ServeMux would otherwise route to the GET handler) while
|
||||
// still being served for safe, read-only routes.
|
||||
@@ -186,20 +367,25 @@ func TestRest_AvatarMounts(t *testing.T) {
|
||||
|
||||
func TestRest_Shutdown(t *testing.T) {
|
||||
srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
|
||||
port := chooseUnusedPort(t)
|
||||
done := make(chan bool)
|
||||
|
||||
// without waiting for channel close at the end goroutine will stay alive after test finish
|
||||
// which would create data race with next test
|
||||
go func() {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
srv.Shutdown()
|
||||
srv.Run("127.0.0.1", port)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
st := time.Now()
|
||||
srv.Run("127.0.0.1", 0)
|
||||
assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
|
||||
<-done
|
||||
defer srv.Shutdown() // a failed readiness wait must not leave srv.Run behind for goleak
|
||||
waitForServerStart(t, port)
|
||||
srv.Shutdown()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(serverStopTimeout):
|
||||
t.Fatal("rest server did not stop after Shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRest_filterComments(t *testing.T) {
|
||||
@@ -218,7 +404,7 @@ func TestRest_filterComments(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRest_RunStaticSSLMode(t *testing.T) {
|
||||
sslPort := chooseRandomUnusedPort()
|
||||
sslPort := chooseUnusedPort(t)
|
||||
srv := Rest{
|
||||
Authenticator: auth.NewService(auth.Opts{
|
||||
AvatarStore: avatar.NewLocalFS("/tmp"),
|
||||
@@ -235,12 +421,12 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
|
||||
RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
|
||||
}
|
||||
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
go func() {
|
||||
srv.Run("", port)
|
||||
}()
|
||||
|
||||
waitForHTTPSServerStart(sslPort)
|
||||
waitForServerStart(t, sslPort, port)
|
||||
|
||||
client := http.Client{
|
||||
// prevent http redirect
|
||||
@@ -273,7 +459,7 @@ func TestRest_RunStaticSSLMode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
|
||||
sslPort := chooseRandomUnusedPort()
|
||||
sslPort := chooseUnusedPort(t)
|
||||
srv := Rest{
|
||||
Authenticator: &auth.Service{},
|
||||
ImageProxy: &proxy.Image{},
|
||||
@@ -284,13 +470,13 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
|
||||
RemarkURL: fmt.Sprintf("https://localhost:%d", sslPort),
|
||||
}
|
||||
|
||||
port := chooseRandomUnusedPort()
|
||||
port := chooseUnusedPort(t)
|
||||
go func() {
|
||||
// can't check https server locally, just only http server
|
||||
srv.Run("", port)
|
||||
}()
|
||||
|
||||
waitForHTTPSServerStart(sslPort)
|
||||
waitForServerStart(t, sslPort, port)
|
||||
|
||||
client := http.Client{
|
||||
// prevent http redirect
|
||||
@@ -387,9 +573,12 @@ func TestRest_frameAncestors(t *testing.T) {
|
||||
client := http.Client{}
|
||||
resp, err := client.Get(ts.URL + "/web/index.html")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "frame-ancestors 'self' https://example.com;")
|
||||
// httptest.Server.Close waits on connections still in use, and a deferred close does not run
|
||||
// until the test ends, so the body has to be released before the server is torn down here
|
||||
require.NoError(t, resp.Body.Close())
|
||||
client.CloseIdleConnections()
|
||||
teardown()
|
||||
|
||||
// test case without frame-ancestors
|
||||
@@ -404,25 +593,11 @@ func TestRest_frameAncestors(t *testing.T) {
|
||||
assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "frame-ancestors *;")
|
||||
}
|
||||
|
||||
// randomPath pick a file or folder name which is not in use for sure
|
||||
func randomPath(tempDir, basename, suffix string) (string, error) {
|
||||
for range 10 {
|
||||
fname := fmt.Sprintf("/%s/%s-%d%s", tempDir, basename, rand.Int31(), suffix)
|
||||
fmt.Printf("fname %q", fname)
|
||||
_, err := os.Stat(fname)
|
||||
if err != nil {
|
||||
return fname, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("cannot create temp file in %s", tempDir)
|
||||
}
|
||||
|
||||
// startupT runs fully configured testing server
|
||||
// srvHook is an optional func to set some Rest param after the creation but prior to Run
|
||||
func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, srv *Rest, teardown func()) {
|
||||
tmp := os.TempDir()
|
||||
testDB, err := randomPath(tmp, "test-remark", ".db")
|
||||
require.NoError(t, err)
|
||||
testDB := filepath.Join(t.TempDir(), "test-remark.db") // per-test dir, removed when the test ends
|
||||
|
||||
_ = os.RemoveAll(tmp + "/ava-remark42")
|
||||
_ = os.RemoveAll(tmp + "/pics-remark42")
|
||||
@@ -503,7 +678,6 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
|
||||
teardown = func() {
|
||||
ts.Close()
|
||||
require.NoError(t, srv.DataService.Close())
|
||||
_ = os.Remove(testDB)
|
||||
_ = os.RemoveAll(tmp + "/ava-remark42")
|
||||
_ = os.RemoveAll(tmp + "/pics-remark42")
|
||||
}
|
||||
@@ -511,6 +685,44 @@ func startupT(t *testing.T, srvHook ...func(srv *Rest)) (ts *httptest.Server, sr
|
||||
return ts, srv, teardown
|
||||
}
|
||||
|
||||
const (
|
||||
// outer bound before a wait is called a hang, generous enough for a loaded CI runner
|
||||
waitTimeout = 30 * time.Second
|
||||
pollInterval = 10 * time.Millisecond
|
||||
|
||||
// budget for a server to stop once asked, tight enough to catch a shutdown that hangs
|
||||
serverStopTimeout = 10 * time.Second
|
||||
|
||||
// connect budget for a single probe, kept off the poll interval so a slow loopback connect
|
||||
// on a loaded runner does not look like a server that is not listening
|
||||
probeDialTimeout = time.Second
|
||||
|
||||
// window to prove something did not happen
|
||||
notifySettle = 300 * time.Millisecond
|
||||
|
||||
// poll interval for waits that issue an HTTP request. the admin routes allow 10 req/s and
|
||||
// the open ones 100 in tests, so this stays below the tighter of the two and the poll
|
||||
// cannot manufacture the 429s it would then have to interpret
|
||||
httpPoll = 150 * time.Millisecond
|
||||
)
|
||||
|
||||
// waitForCount blocks until got reaches want, failing the test with the last value it saw.
|
||||
// for work that is delivered asynchronously, such as notifications reaching a mock destination
|
||||
func waitForCount(t *testing.T, want int, got func() int, msgAndArgs ...any) {
|
||||
t.Helper()
|
||||
require.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||
assert.Equal(c, want, got(), msgAndArgs...)
|
||||
}, waitTimeout, pollInterval)
|
||||
}
|
||||
|
||||
// waitForCountSettled waits for got to reach want and then holds it there, so a delivery
|
||||
// arriving late is caught rather than passing because the count was read the instant it matched
|
||||
func waitForCountSettled(t *testing.T, want int, got func() int, msgAndArgs ...any) {
|
||||
t.Helper()
|
||||
waitForCount(t, want, got, msgAndArgs...)
|
||||
require.Never(t, func() bool { return got() != want }, notifySettle, pollInterval, msgAndArgs...)
|
||||
}
|
||||
|
||||
// fake auth middleware make user authenticated and uses query's fake_id for ID and fake_name for Name
|
||||
func fakeAuth(next http.Handler) http.Handler {
|
||||
fn := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -534,7 +746,7 @@ func get(t *testing.T, url string) (response string, statusCode int) {
|
||||
return string(body), r.StatusCode
|
||||
}
|
||||
|
||||
func sendReq(_ *testing.T, r *http.Request, tkn string) (*http.Response, error) {
|
||||
func sendReq(r *http.Request, tkn string) (*http.Response, error) {
|
||||
client := http.Client{Timeout: 5 * time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
if tkn != "" {
|
||||
@@ -616,7 +828,6 @@ func addCommentGetCreatedTime(t *testing.T, c store.Comment, ts *httptest.Server
|
||||
crResp := R.JSON{}
|
||||
err = json.Unmarshal(b, &crResp)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Nanosecond * 10)
|
||||
created, err = time.Parse(time.RFC3339, crResp["time"].(string))
|
||||
require.NoError(t, err)
|
||||
return crResp["id"].(string), created
|
||||
@@ -628,37 +839,41 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
|
||||
}
|
||||
|
||||
func requireAdminOnly(t *testing.T, req *http.Request) {
|
||||
resp, err := sendReq(t, req, "") // no-auth user
|
||||
resp, err := sendReq(req, "") // no-auth user
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
|
||||
resp, err = sendReq(t, req, devToken) // non-admin user
|
||||
resp, err = sendReq(req, devToken) // non-admin user
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
}
|
||||
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for range 10 {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
return port
|
||||
}
|
||||
|
||||
func waitForHTTPSServerStart(port int) {
|
||||
// wait for up to 3 seconds for HTTPS server to start
|
||||
for range 300 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
conn, _ := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
|
||||
if conn != nil {
|
||||
// waitForServerStart blocks until something accepts on every listed port, failing the test
|
||||
// naming the port that never came up
|
||||
func waitForServerStart(t *testing.T, ports ...int) {
|
||||
t.Helper()
|
||||
for _, port := range ports {
|
||||
require.Eventually(t, func() bool {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), probeDialTimeout)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
break
|
||||
}
|
||||
return true
|
||||
}, waitTimeout, pollInterval, "server on port %d didn't start", port)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -667,5 +882,9 @@ func TestMain(m *testing.M) {
|
||||
m,
|
||||
// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
|
||||
goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
|
||||
// regexp2, pulled in by chroma for syntax highlighting, keeps one shared clock goroutine
|
||||
// alive for up to a second after the last match with a timeout, sleeping in 100ms ticks.
|
||||
// it ends on its own, but a binary that finishes inside that window is reported as leaking
|
||||
goleak.IgnoreAnyFunction("github.com/dlclark/regexp2/v2.runClock"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -14,22 +14,29 @@ import (
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
)
|
||||
|
||||
// rssPubTime returns a second-aligned base timestamp and formats it the way the feed does, so
|
||||
// comment pubDates are pinned rather than dependent on when in the second the test runs.
|
||||
func rssPubTime() (base time.Time, pubDate string) {
|
||||
base = time.Now().Truncate(time.Second)
|
||||
return base, base.Format(time.RFC1123Z)
|
||||
}
|
||||
|
||||
func TestServer_RssPost(t *testing.T) {
|
||||
ts, rst, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
waitOnSecChange()
|
||||
base, pubDate := rssPubTime()
|
||||
|
||||
c1 := store.Comment{
|
||||
ID: "1234567890",
|
||||
Text: "test 123",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
ID: "1234567890",
|
||||
Text: "test 123",
|
||||
Timestamp: base,
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
}
|
||||
id1, err := rst.DataService.Create(c1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "1234567890", id1)
|
||||
pubDate := time.Now().Format(time.RFC1123Z)
|
||||
|
||||
res, code := get(t, ts.URL+"/api/v1/rss/post?site=remark42&url=https://radio-t.com/blah1")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
@@ -63,21 +70,21 @@ func TestServer_RssSite(t *testing.T) {
|
||||
ts, rst, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
waitOnSecChange()
|
||||
|
||||
pubDate := time.Now().Format(time.RFC1123Z)
|
||||
base, pubDate := rssPubTime()
|
||||
|
||||
c1 := store.Comment{
|
||||
ID: "comment-id-1",
|
||||
Text: "test 123",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
ID: "comment-id-1",
|
||||
Text: "test 123",
|
||||
Timestamp: base,
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
}
|
||||
c2 := store.Comment{
|
||||
ID: "comment-id-2",
|
||||
Text: "xyz test",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
ID: "comment-id-2",
|
||||
Text: "xyz test",
|
||||
Timestamp: base.Add(time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
}
|
||||
|
||||
_, err := rst.DataService.Create(c1)
|
||||
@@ -126,22 +133,22 @@ func TestServer_RssWithReply(t *testing.T) {
|
||||
ts, rst, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
waitOnSecChange()
|
||||
|
||||
pubDate := time.Now().Format(time.RFC1123Z)
|
||||
base, pubDate := rssPubTime()
|
||||
|
||||
c1 := store.Comment{
|
||||
ID: "comment-id-1",
|
||||
Text: "test 123",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
ID: "comment-id-1",
|
||||
Text: "test 123",
|
||||
Timestamp: base,
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
}
|
||||
c2 := store.Comment{
|
||||
ID: "comment-id-2",
|
||||
ParentID: "comment-id-1",
|
||||
Text: "xyz test",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
ID: "comment-id-2",
|
||||
ParentID: "comment-id-1",
|
||||
Text: "xyz test",
|
||||
Timestamp: base.Add(time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "remark42"},
|
||||
User: store.User{ID: "u1", Name: "developer one"},
|
||||
}
|
||||
|
||||
_, err := rst.DataService.Create(c1)
|
||||
@@ -186,42 +193,45 @@ func TestServer_RssReplies(t *testing.T) {
|
||||
ts, srv, teardown := startupT(t)
|
||||
defer teardown()
|
||||
|
||||
waitOnSecChange()
|
||||
|
||||
pubDate := time.Now().Format(time.RFC1123Z)
|
||||
base, pubDate := rssPubTime()
|
||||
|
||||
c1 := store.Comment{
|
||||
ID: "comment-1",
|
||||
Text: "c1",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user1", Name: "user1"},
|
||||
ID: "comment-1",
|
||||
Text: "c1",
|
||||
Timestamp: base,
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user1", Name: "user1"},
|
||||
}
|
||||
c2 := store.Comment{
|
||||
ID: "comment-2",
|
||||
Text: "reply to c1 from user2",
|
||||
ParentID: "comment-1",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user2", Name: "user2"},
|
||||
ID: "comment-2",
|
||||
Text: "reply to c1 from user2",
|
||||
ParentID: "comment-1",
|
||||
Timestamp: base.Add(time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user2", Name: "user2"},
|
||||
}
|
||||
c3 := store.Comment{
|
||||
ID: "comment-3",
|
||||
Text: "reply to c1 from user3",
|
||||
ParentID: "comment-1",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user3", Name: "user3"},
|
||||
ID: "comment-3",
|
||||
Text: "reply to c1 from user3",
|
||||
ParentID: "comment-1",
|
||||
Timestamp: base.Add(2 * time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "user3", Name: "user3"},
|
||||
}
|
||||
c4 := store.Comment{
|
||||
ID: "comment-4",
|
||||
Text: "reply to c2 from developer one",
|
||||
ParentID: "comment-2",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "dev", Name: "developer one"},
|
||||
ID: "comment-4",
|
||||
Text: "reply to c2 from developer one",
|
||||
ParentID: "comment-2",
|
||||
Timestamp: base.Add(3 * time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "dev", Name: "developer one"},
|
||||
}
|
||||
c5 := store.Comment{
|
||||
ID: "comment-5",
|
||||
Text: "developer one",
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "dev", Name: "developer one"},
|
||||
ID: "comment-5",
|
||||
Text: "developer one",
|
||||
Timestamp: base.Add(4 * time.Millisecond),
|
||||
Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "remark42"},
|
||||
User: store.User{ID: "dev", Name: "developer one"},
|
||||
}
|
||||
|
||||
_, err := srv.DataService.Create(c1)
|
||||
@@ -270,12 +280,6 @@ func TestServer_RssReplies(t *testing.T) {
|
||||
assert.Equal(t, http.StatusBadRequest, code)
|
||||
}
|
||||
|
||||
func waitOnSecChange() {
|
||||
for time.Now().Nanosecond() >= 100000000 {
|
||||
time.Sleep(10 * time.Nanosecond)
|
||||
}
|
||||
}
|
||||
|
||||
// clean formatting, i.e. multiple spaces, \t, \n
|
||||
func cleanRssFormatting(expected, actual string) (cleanExp, cleanAct string) {
|
||||
reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// webFiles serves /web from two sources: a name present in the frontend build is served from there,
|
||||
// and any other name from the assets embedded in the binary.
|
||||
type webFiles struct {
|
||||
frontend fs.FS
|
||||
embedded fs.FS
|
||||
}
|
||||
|
||||
// Open resolves the name against both sources, and answers a missing .js with the .mjs sibling.
|
||||
// The build stopped emitting .js while integrations still request it; the bundles carry no module
|
||||
// syntax, so the same bytes serve both names.
|
||||
func (w webFiles) Open(name string) (fs.File, error) {
|
||||
// fs.ValidPath alone is not enough: it accepts names the operating system rejects, NUL among
|
||||
// them, and os.DirFS turns those into fs.ErrInvalid, which renders as 500 rather than 404
|
||||
if _, err := filepath.Localize(name); err != nil || !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
f, err := w.open(name)
|
||||
if err == nil {
|
||||
return f, nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrNotExist) || !strings.HasSuffix(name, ".js") {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
alias, aliasErr := w.open(strings.TrimSuffix(name, ".js") + ".mjs")
|
||||
if aliasErr == nil {
|
||||
return alias, nil
|
||||
}
|
||||
if !errors.Is(aliasErr, fs.ErrNotExist) {
|
||||
return nil, aliasErr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open looks the name up in the frontend build first. Only a missing file falls through to the
|
||||
// embedded assets; every other error is returned so an unreadable file keeps reporting as one
|
||||
// rather than being replaced by the embedded copy or reported as missing.
|
||||
func (w webFiles) open(name string) (fs.File, error) {
|
||||
f, err := w.frontend.Open(name)
|
||||
if err == nil {
|
||||
return f, nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
if name == "." {
|
||||
// the embedded set is a flat list of files; only the frontend build answers for the
|
||||
// directory itself, so a missing web root reports as missing rather than listing them
|
||||
return nil, err
|
||||
}
|
||||
return w.embedded.Open(name)
|
||||
}
|
||||
|
||||
// emptyFS stands in for a frontend source that could not be opened, so a misconfigured one serves
|
||||
// nothing instead of panicking or serving the build at paths it does not belong at
|
||||
type emptyFS struct{}
|
||||
|
||||
func (emptyFS) Open(name string) (fs.File, error) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
// remarkURLPlaceholder is what the frontend build carries wherever the instance URL belongs. The
|
||||
// bundler cannot know that URL, so it emits this marker and every distribution fills it in: the
|
||||
// docker image rewrites the files under the web root at container start, and the binary, which
|
||||
// serves the build embedded in itself and has nothing to rewrite, does it here.
|
||||
const remarkURLPlaceholder = "{% REMARK_URL %}"
|
||||
|
||||
// templatedFS fills the instance URL into the files carrying the placeholder. Without it the
|
||||
// binary serves whatever the build baked in, which is a host no visitor can reach, and the widget
|
||||
// falls back to it whenever a page omits remark_config.host.
|
||||
type templatedFS struct {
|
||||
fs fs.FS
|
||||
remarkURL string
|
||||
}
|
||||
|
||||
// Open substitutes in the file types the frontend templates, and hands everything else through
|
||||
// untouched so images and stylesheets keep streaming from their original source
|
||||
func (t templatedFS) Open(name string) (fs.File, error) {
|
||||
f, err := t.fs.Open(name)
|
||||
if err != nil || !templatedName(name) {
|
||||
return f, err
|
||||
}
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil || info.IsDir() {
|
||||
return f, err
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(f)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body = bytes.ReplaceAll(body, []byte(remarkURLPlaceholder), []byte(t.remarkURL))
|
||||
return &memFile{Reader: bytes.NewReader(body), info: sizedInfo{FileInfo: info, size: int64(len(body))}}, nil
|
||||
}
|
||||
|
||||
// templatedName reports whether the frontend templates this file type. It mirrors the set the
|
||||
// docker image rewrites, so both distributions substitute in the same files
|
||||
func templatedName(name string) bool {
|
||||
switch filepath.Ext(name) {
|
||||
case ".html", ".js", ".mjs":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// memFile is a substituted file held in memory. The file server needs a seeker to answer range
|
||||
// requests and to sniff a content type, which a substituted body no longer has on disk
|
||||
type memFile struct {
|
||||
*bytes.Reader
|
||||
info fs.FileInfo
|
||||
}
|
||||
|
||||
func (f *memFile) Stat() (fs.FileInfo, error) { return f.info, nil }
|
||||
func (f *memFile) Close() error { return nil }
|
||||
|
||||
// sizedInfo reports the length after substitution. The file server writes Content-Length from it,
|
||||
// so reporting the length on disk would truncate the response or leave the client waiting
|
||||
type sizedInfo struct {
|
||||
fs.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (i sizedInfo) Size() int64 { return i.size }
|
||||
@@ -0,0 +1,343 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/go-pkgz/routegroup"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/webassets"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWebFiles_Open(t *testing.T) {
|
||||
frontend := fstest.MapFS{
|
||||
"both.html": {Data: []byte("from the frontend build")},
|
||||
"only-frontend.html": {Data: []byte("frontend only")},
|
||||
}
|
||||
embedded := fstest.MapFS{
|
||||
"both.html": {Data: []byte("from the embedded assets")},
|
||||
"only-embedded.html": {Data: []byte("embedded only")},
|
||||
}
|
||||
w := webFiles{frontend: frontend, embedded: embedded}
|
||||
|
||||
tbl := []struct {
|
||||
name string
|
||||
lookup string
|
||||
want string
|
||||
wantErr error
|
||||
}{
|
||||
{name: "present in both is served from the frontend build", lookup: "both.html", want: "from the frontend build"},
|
||||
{name: "frontend only", lookup: "only-frontend.html", want: "frontend only"},
|
||||
{name: "embedded only", lookup: "only-embedded.html", want: "embedded only"},
|
||||
{name: "missing in both", lookup: "neither.html", wantErr: fs.ErrNotExist},
|
||||
}
|
||||
|
||||
for _, tt := range tbl {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f, err := w.Open(tt.lookup)
|
||||
if tt.wantErr != nil {
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(f)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, string(b))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebFiles_OpenJSAlias(t *testing.T) {
|
||||
frontend := fstest.MapFS{
|
||||
"embed.mjs": {Data: []byte("module embed")},
|
||||
"counter.js": {Data: []byte("operator's own counter")},
|
||||
"counter.mjs": {Data: []byte("module counter")},
|
||||
"widget.mjs": {Data: []byte("module widget")},
|
||||
}
|
||||
embedded := fstest.MapFS{
|
||||
"legacy.mjs": {Data: []byte("module legacy")},
|
||||
"widget.js": {Data: []byte("embedded widget")},
|
||||
}
|
||||
w := webFiles{frontend: frontend, embedded: embedded}
|
||||
|
||||
tbl := []struct {
|
||||
name string
|
||||
lookup string
|
||||
want string
|
||||
wantErr error
|
||||
}{
|
||||
{name: "missing js served from the mjs sibling", lookup: "embed.js", want: "module embed"},
|
||||
{name: "alias reaches the embedded assets too", lookup: "legacy.js", want: "module legacy"},
|
||||
{name: "a real js file wins over its sibling", lookup: "counter.js", want: "operator's own counter"},
|
||||
{name: "an embedded js wins over a frontend sibling", lookup: "widget.js", want: "embedded widget"},
|
||||
{name: "mjs is still served directly", lookup: "embed.mjs", want: "module embed"},
|
||||
{name: "neither name present", lookup: "absent.js", wantErr: fs.ErrNotExist},
|
||||
{name: "only js aliases, not other extensions", lookup: "embed.html", wantErr: fs.ErrNotExist},
|
||||
}
|
||||
|
||||
for _, tt := range tbl {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f, err := w.Open(tt.lookup)
|
||||
if tt.wantErr != nil {
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(f)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, string(b))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebFiles_OpenJSAliasNamesTheRequestedFile(t *testing.T) {
|
||||
w := webFiles{frontend: fstest.MapFS{}, embedded: fstest.MapFS{}}
|
||||
|
||||
_, err := w.Open("absent.js")
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
assert.Contains(t, err.Error(), "absent.js")
|
||||
assert.NotContains(t, err.Error(), "absent.mjs")
|
||||
}
|
||||
|
||||
func TestWebFiles_OpenJSAliasUnreadableSibling(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root ignores file permissions")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "embed.mjs"), []byte("module embed"), 0o000))
|
||||
w := webFiles{frontend: os.DirFS(dir), embedded: fstest.MapFS{}}
|
||||
|
||||
f, err := w.Open("embed.js")
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, fs.ErrPermission)
|
||||
assert.NotErrorIs(t, err, fs.ErrNotExist, "an unreadable sibling must not render as 404")
|
||||
if err == nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmptyFS_ServesNothing pins the stand-in used when the frontend source cannot be opened:
|
||||
// every name must report as missing rather than panicking, since it backs a nil-free fallback.
|
||||
func TestEmptyFS_ServesNothing(t *testing.T) {
|
||||
for _, name := range []string{".", "index.html", "web/index.html"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f, err := emptyFS{}.Open(name)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
assert.Nil(t, f)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebFiles_EmptyFrontendFallsThrough covers the shape routes() builds when fs.Sub refuses:
|
||||
// the embedded assets must still answer even though the frontend source serves nothing.
|
||||
func TestWebFiles_EmptyFrontendFallsThrough(t *testing.T) {
|
||||
w := webFiles{frontend: emptyFS{}, embedded: webassets.FS}
|
||||
|
||||
want, err := fs.ReadFile(webassets.FS, "privacy.html")
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := w.Open("privacy.html")
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
got, err := io.ReadAll(f)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(want), string(got))
|
||||
}
|
||||
|
||||
// TestWebFiles_OpenRootIsNotListed keeps the embedded assets from being browsable: they answer
|
||||
// for their own names only, so a web root that has gone missing reports as missing.
|
||||
func TestWebFiles_OpenRootIsNotListed(t *testing.T) {
|
||||
w := webFiles{frontend: os.DirFS(filepath.Join(t.TempDir(), "absent")), embedded: webassets.FS}
|
||||
|
||||
f, err := w.Open(".")
|
||||
require.Error(t, err, "the embedded assets must not answer for the directory itself")
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
if err == nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
|
||||
// the assets themselves still serve
|
||||
f, err = w.Open("privacy.html")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
}
|
||||
|
||||
// TestWebFiles_OpenInvalidName pins that a name fs rejects reports as missing rather than invalid.
|
||||
// os.DirFS returns fs.ErrInvalid for these, which http.FileServer renders as 500, so the check has
|
||||
// to happen before the lookup. A memory filesystem cannot show this: it reports missing either way.
|
||||
func TestWebFiles_OpenInvalidName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "privacy.html"), []byte("frontend"), 0o600))
|
||||
w := webFiles{frontend: os.DirFS(dir), embedded: webassets.FS}
|
||||
|
||||
for _, name := range []string{"../escape.html", "/etc/passwd", "a\x00b.html", "./privacy.html"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f, err := w.Open(name)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
assert.NotErrorIs(t, err, fs.ErrInvalid, "an invalid name must not surface as 500")
|
||||
if err == nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebFiles_OpenUnreadableFrontendFile pins the rule that only a missing file falls through:
|
||||
// a frontend file that cannot be read must report that, not be masked by the embedded copy.
|
||||
func TestWebFiles_OpenUnreadableFrontendFile(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root ignores file permissions")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "privacy.html"), []byte("operator's own"), 0o000))
|
||||
|
||||
w := webFiles{
|
||||
frontend: os.DirFS(dir),
|
||||
embedded: fstest.MapFS{"privacy.html": {Data: []byte("built in")}},
|
||||
}
|
||||
|
||||
f, err := w.Open("privacy.html")
|
||||
require.Error(t, err, "an unreadable frontend file must not be replaced by the embedded copy")
|
||||
assert.NotErrorIs(t, err, fs.ErrNotExist, "the error must stay a permission error so it does not render as 404")
|
||||
assert.ErrorIs(t, err, fs.ErrPermission)
|
||||
if err == nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplatedFS_SubstitutesTheInstanceURL(t *testing.T) {
|
||||
const placeholder = "host: '" + remarkURLPlaceholder + "'"
|
||||
source := fstest.MapFS{
|
||||
"iframe.html": {Data: []byte(placeholder)},
|
||||
"embed.mjs": {Data: []byte(placeholder)},
|
||||
"embed.js": {Data: []byte(placeholder)},
|
||||
"remark.css": {Data: []byte(placeholder)},
|
||||
"nothing.html": {Data: []byte("no marker here")},
|
||||
}
|
||||
tfs := templatedFS{fs: source, remarkURL: "https://remark.example.com"}
|
||||
|
||||
tbl := []struct {
|
||||
name string
|
||||
want string
|
||||
}{
|
||||
{"iframe.html", "host: 'https://remark.example.com'"},
|
||||
{"embed.mjs", "host: 'https://remark.example.com'"},
|
||||
{"embed.js", "host: 'https://remark.example.com'"},
|
||||
// the docker image rewrites html, js and mjs and nothing else, and a stylesheet carrying
|
||||
// the marker would mean the frontend started templating a file type this does not cover
|
||||
{"remark.css", placeholder},
|
||||
{"nothing.html", "no marker here"},
|
||||
}
|
||||
|
||||
for _, tt := range tbl {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f, err := tfs.Open(tt.name)
|
||||
require.NoError(t, err)
|
||||
defer func() { assert.NoError(t, f.Close()) }()
|
||||
|
||||
body, err := io.ReadAll(f)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, string(body))
|
||||
|
||||
info, err := f.Stat()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(tt.want)), info.Size(),
|
||||
"the size has to be the substituted one, or the response is truncated or left hanging")
|
||||
assert.Equal(t, tt.name, info.Name())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplatedFS_PassesErrorsThrough(t *testing.T) {
|
||||
tfs := templatedFS{fs: fstest.MapFS{}, remarkURL: "https://remark.example.com"}
|
||||
|
||||
_, err := tfs.Open("absent.html")
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
}
|
||||
|
||||
// TestRest_FileServerFillsInTheInstanceURL covers the reason templatedFS exists: the binary serves
|
||||
// the frontend build embedded in itself, and nothing else fills the placeholder in for it.
|
||||
func TestRest_FileServerFillsInTheInstanceURL(t *testing.T) {
|
||||
frontend := fstest.MapFS{
|
||||
"embed.mjs": {Data: []byte("host=\"" + remarkURLPlaceholder + "\"")},
|
||||
"logo.svg": {Data: []byte(remarkURLPlaceholder)},
|
||||
"plain.html": {Data: []byte("nothing to fill in")},
|
||||
}
|
||||
router := routegroup.New(http.NewServeMux())
|
||||
addFileServer(router, frontend, filepath.Join(t.TempDir(), "absent"), "test-version", "https://remark.example.com")
|
||||
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
t.Run("the bundle carries the configured url", func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/embed.mjs")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, `host="https://remark.example.com"`, body)
|
||||
})
|
||||
|
||||
t.Run("the legacy js name carries it too", func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/embed.js")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, `host="https://remark.example.com"`, body)
|
||||
})
|
||||
|
||||
t.Run("other types are served untouched", func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/logo.svg")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, remarkURLPlaceholder, body)
|
||||
})
|
||||
|
||||
t.Run("a file without the marker is unchanged", func(t *testing.T) {
|
||||
body, code := get(t, ts.URL+"/web/plain.html")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, "nothing to fill in", body)
|
||||
})
|
||||
}
|
||||
|
||||
// TestRest_FileServerEtagVariesWithTheInstanceURL covers the case this substitution exists for. An
|
||||
// operator who notices the widget is addressed to the wrong host corrects REMARK_URL and restarts,
|
||||
// and the binary and so the version is unchanged. If the validator ignores remarkURL the client
|
||||
// revalidates, gets 304 and keeps the bundle pointing at the old host. Cache-Control is no-cache,
|
||||
// so it revalidates every time and never ages out of that state.
|
||||
func TestRest_FileServerEtagVariesWithTheInstanceURL(t *testing.T) {
|
||||
frontend := fstest.MapFS{"embed.mjs": {Data: []byte("host=\"" + remarkURLPlaceholder + "\"")}}
|
||||
|
||||
etagFor := func(remarkURL string) string {
|
||||
router := routegroup.New(http.NewServeMux())
|
||||
addFileServer(router, frontend, filepath.Join(t.TempDir(), "absent"), "test-version", remarkURL)
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
resp, err := http.Get(ts.URL + "/web/embed.mjs")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
return resp.Header.Get("Etag")
|
||||
}
|
||||
|
||||
first := etagFor("https://old.example.com")
|
||||
second := etagFor("https://new.example.com")
|
||||
|
||||
require.NotEmpty(t, first, "the file server has to send a validator at all")
|
||||
assert.NotEqual(t, first, second,
|
||||
"same version and same path, different instance url: the validator has to change or the "+
|
||||
"client keeps a bundle addressed to the old host")
|
||||
}
|
||||
@@ -813,8 +813,8 @@ func imgHTTPTestsServer(t *testing.T) *httptest.Server {
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/image/img-slow.png" {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
w.WriteHeader(500)
|
||||
// hold the response until the proxy gives up on its own timeout
|
||||
<-r.Context().Done()
|
||||
return
|
||||
}
|
||||
t.Log("http img request - not found", r.URL)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -151,8 +152,8 @@ func (c *Comment) Snippet(limit int) string {
|
||||
}
|
||||
snippet := []rune(cleanText)[:limit]
|
||||
// go back in snippet and found the first space
|
||||
for i := len(snippet) - 1; i >= 0; i-- {
|
||||
if snippet[i] == ' ' {
|
||||
for i, s := range slices.Backward(snippet) {
|
||||
if s == ' ' {
|
||||
snippet = snippet[:i]
|
||||
break
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
berrors "go.etcd.io/bbolt/errors"
|
||||
|
||||
@@ -415,14 +414,14 @@ func (b *BoltDB) Delete(req DeleteRequest) error {
|
||||
|
||||
// Close boltdb store
|
||||
func (b *BoltDB) Close() error {
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
for site, db := range b.dbs {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("can't close site %s: %w", site, err))
|
||||
errs = append(errs, fmt.Errorf("can't close site %s: %w", site, err))
|
||||
}
|
||||
}
|
||||
return errs.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Last returns up to max last comments for given siteID
|
||||
@@ -1019,7 +1018,7 @@ func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
|
||||
if res, ok := b.dbs[siteID]; ok {
|
||||
return res, nil
|
||||
}
|
||||
return nil, fmt.Errorf("site %q not found", siteID)
|
||||
return nil, fmt.Errorf("site %q %w", siteID, ErrSiteNotFound)
|
||||
}
|
||||
|
||||
// makeRef creates reference combining url and comment id
|
||||
|
||||
@@ -4,6 +4,7 @@ package engine
|
||||
// Includes default implementation with boltdb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -11,6 +12,10 @@ import (
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
)
|
||||
|
||||
// ErrSiteNotFound is returned by engines when the requested site does not exist.
|
||||
// Its message is "not found" so wrapping it as `site %q %w` reads "site \"x\" not found".
|
||||
var ErrSiteNotFound = errors.New("not found")
|
||||
|
||||
// NOTE: matryer/moq should be installed globally and works with `go generate ./...`
|
||||
//go:generate moq --out engine_mock.go . Interface
|
||||
|
||||
|
||||
@@ -147,8 +147,8 @@ func (f *FileSystem) Cleanup(_ context.Context, ttl time.Duration) error {
|
||||
age := time.Since(info.ModTime())
|
||||
if age > (ttl + 100*time.Millisecond) { // delay cleanup triggering to allow commit
|
||||
log.Printf("[INFO] remove staging image %s, age %v", fpath, age)
|
||||
rmErr := os.Remove(fpath) //nolint:gosec // staging dir is server-only, no untrusted symlinks land here
|
||||
_ = os.Remove(path.Dir(fpath)) //nolint:gosec // same staging dir
|
||||
rmErr := os.Remove(fpath) //nolint:gosec // staging dir is server-only, no untrusted symlinks land here
|
||||
_ = os.Remove(path.Dir(fpath)) //nolint:gosec // same staging dir
|
||||
return rmErr
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -215,15 +215,24 @@ func TestFsStore_Cleanup(t *testing.T) {
|
||||
return img
|
||||
}
|
||||
|
||||
// age is read from the file's modification time, so every file gets its mtime stamped right
|
||||
// before each call: far past the ttl for the ones meant to go, at now for the ones meant to
|
||||
// survive, leaving no window for a stalled runner to age a survivor into the wrong bucket
|
||||
const ttl = 300 * time.Millisecond
|
||||
age := func(file string, d time.Duration) {
|
||||
mtime := time.Now().Add(-d)
|
||||
require.NoError(t, os.Chtimes(file, mtime, mtime))
|
||||
}
|
||||
|
||||
// save 3 images to staging
|
||||
img1 := save("blah_ff1.png", "user1")
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
img2 := save("blah_ff2.png", "user1")
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
img3 := save("blah_ff3.png", "user2")
|
||||
|
||||
time.Sleep(200 * time.Millisecond) // make first image expired
|
||||
err := svc.Cleanup(context.Background(), time.Millisecond*300)
|
||||
age(img1, time.Hour) // past the ttl, collected
|
||||
age(img2, 0) // fresh, survives
|
||||
age(img3, 0)
|
||||
err := svc.Cleanup(context.Background(), ttl)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = os.Stat(img1)
|
||||
@@ -242,10 +251,11 @@ func TestFsStore_Cleanup(t *testing.T) {
|
||||
_, err = os.Stat(img3)
|
||||
assert.NoError(t, err, "file on staging")
|
||||
|
||||
time.Sleep(200 * time.Millisecond) // make all images expired
|
||||
age(img2, time.Hour)
|
||||
age(img3, time.Hour)
|
||||
err = svc.ResetCleanupTimer("user2/blah_ff3.png") // reset the time to cleanup for third image
|
||||
assert.NoError(t, err)
|
||||
err = svc.Cleanup(context.Background(), time.Millisecond*300)
|
||||
err = svc.Cleanup(context.Background(), ttl)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = os.Stat(img2)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha1" //nolint:gosec // not used for cryptography
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif" // register gif decoder
|
||||
@@ -27,7 +28,6 @@ import (
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/rs/xid"
|
||||
"golang.org/x/image/draw"
|
||||
_ "golang.org/x/image/webp" // register webp decoder so DecodeConfig accepts what readAndValidateImage allows
|
||||
@@ -43,8 +43,8 @@ type Service struct {
|
||||
wg sync.WaitGroup
|
||||
submitCh chan submitReq
|
||||
once sync.Once
|
||||
term int32 // term value used atomically to detect emergency termination
|
||||
submitCount int32 // atomic increment for counting submitted images
|
||||
term atomic.Int32 // term value used atomically to detect emergency termination
|
||||
submitCount atomic.Int32 // atomic increment for counting submitted images
|
||||
}
|
||||
|
||||
// ServiceParams contains externally adjustable parameters of Service
|
||||
@@ -91,14 +91,14 @@ func NewService(s Store, p ServiceParams) *Service {
|
||||
|
||||
// Commit multiple ids immediately
|
||||
func (s *Service) Commit(idsFn func() []string) error {
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
for _, id := range idsFn() {
|
||||
err := s.store.Commit(id)
|
||||
if err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("failed to commit image %s: %w", id, err))
|
||||
errs = append(errs, fmt.Errorf("failed to commit image %s: %w", id, err))
|
||||
}
|
||||
}
|
||||
return errs.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Submit multiple ids via function for delayed commit
|
||||
@@ -113,7 +113,7 @@ func (s *Service) Submit(idsFn func() []string) {
|
||||
s.wg.Go(func() {
|
||||
for req := range s.submitCh {
|
||||
// wait for EditDuration expiration with emergency pass on term
|
||||
for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.EditDuration {
|
||||
for s.term.Load() == 0 && time.Since(req.TS) <= s.EditDuration {
|
||||
time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
|
||||
}
|
||||
err := s.Commit(req.idsFn)
|
||||
@@ -121,13 +121,13 @@ func (s *Service) Submit(idsFn func() []string) {
|
||||
log.Printf("[WARN] image commit error %v", err)
|
||||
}
|
||||
|
||||
atomic.AddInt32(&s.submitCount, -1)
|
||||
s.submitCount.Add(-1)
|
||||
}
|
||||
log.Printf("[INFO] image submitter terminated")
|
||||
})
|
||||
})
|
||||
|
||||
atomic.AddInt32(&s.submitCount, 1)
|
||||
s.submitCount.Add(1)
|
||||
|
||||
// reset cleanup timer before submitting the images
|
||||
// to prevent them from being cleaned up while waiting for EditDuration to expire
|
||||
@@ -196,14 +196,14 @@ func (s *Service) Close(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if atomic.LoadInt32(&s.submitCount) == 0 {
|
||||
if s.submitCount.Load() == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
|
||||
s.term.Store(1) // enforce non-delayed commits for all ids left in submitCh
|
||||
waitForTerm(ctx)
|
||||
|
||||
if s.submitCh != nil {
|
||||
|
||||
@@ -41,13 +41,15 @@ func TestService_SaveAndLoad(t *testing.T) {
|
||||
assert.Equal(t, "test_id", store.LoadCalls()[0].ID)
|
||||
}
|
||||
|
||||
// the resized dimensions are what resize promises; the encoded length is whatever the compressor
|
||||
// in the toolchain happens to produce, and pinning it fails on a go release that changes it
|
||||
func TestService_Resize(t *testing.T) {
|
||||
img, err := readAndValidateImage(gopherPNG(), 1500)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1462, len(img))
|
||||
assert.NotEmpty(t, img)
|
||||
|
||||
img = resize(img, 32, 32)
|
||||
assert.Equal(t, 1135, len(img))
|
||||
assertImageFits(t, img, 32, 32)
|
||||
}
|
||||
|
||||
func TestService_ResizeJpeg(t *testing.T) {
|
||||
@@ -57,10 +59,24 @@ func TestService_ResizeJpeg(t *testing.T) {
|
||||
|
||||
img, err := readAndValidateImage(fh, 32000)
|
||||
assert.NoError(t, err)
|
||||
assert.InDelta(t, 16756, len(img), 100)
|
||||
assert.NotEmpty(t, img)
|
||||
|
||||
img = resize(img, 400, 300)
|
||||
assert.InDelta(t, 10913, len(img), 100)
|
||||
assertImageFits(t, img, 400, 300)
|
||||
}
|
||||
|
||||
// assertImageFits decodes the image and checks it is inside the box resize was given, and that it
|
||||
// touches one side of it, which is what fitting to a box rather than merely shrinking means
|
||||
func assertImageFits(t *testing.T, data []byte, limitW, limitH int) {
|
||||
t.Helper()
|
||||
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
require.NoError(t, err, "the resized image does not decode")
|
||||
|
||||
assert.LessOrEqual(t, cfg.Width, limitW, "wider than the box it was resized into")
|
||||
assert.LessOrEqual(t, cfg.Height, limitH, "taller than the box it was resized into")
|
||||
assert.True(t, cfg.Width == limitW || cfg.Height == limitH,
|
||||
"%dx%d touches neither side of the %dx%d box, so it was not fitted to it", cfg.Width, cfg.Height, limitW, limitH)
|
||||
}
|
||||
|
||||
func TestService_SaveTooLarge(t *testing.T) {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
@@ -14,7 +15,6 @@ import (
|
||||
"github.com/go-pkgz/lcw/v2"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
bf "github.com/russross/blackfriday/v2"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
@@ -128,6 +128,7 @@ func (s *DataStore) FindSince(locator store.Locator, sortMethod string, user sto
|
||||
}
|
||||
|
||||
changedSort := false
|
||||
flags := s.newUserFlagCache()
|
||||
// sets votes controversy for comments added prior to #274
|
||||
// also sanitizes locator.URL for comments added prior to #927
|
||||
for i, c := range comments {
|
||||
@@ -137,7 +138,7 @@ func (s *DataStore) FindSince(locator store.Locator, sortMethod string, user sto
|
||||
changedSort = true
|
||||
}
|
||||
}
|
||||
comments[i] = s.alterComment(c, user)
|
||||
comments[i] = s.alterCommentCached(c, user, flags)
|
||||
}
|
||||
|
||||
// resort commits if altered
|
||||
@@ -249,18 +250,18 @@ func (s *DataStore) ResubmitStagingImages(sites []string) error {
|
||||
if ts.IsZero() {
|
||||
return nil
|
||||
}
|
||||
result := new(multierror.Error)
|
||||
var errs []error
|
||||
for _, site := range sites {
|
||||
locator := store.Locator{SiteID: site}
|
||||
comments, err := s.FindSince(locator, "time", store.User{}, ts)
|
||||
if err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("problem finding comments for site %s: %w", site, err))
|
||||
errs = append(errs, fmt.Errorf("problem finding comments for site %s: %w", site, err))
|
||||
}
|
||||
for _, c := range comments {
|
||||
s.submitImages(c)
|
||||
}
|
||||
}
|
||||
return result.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// submitImages initiated delayed commit of all images from the comment uploaded to remark42
|
||||
@@ -915,32 +916,32 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
|
||||
|
||||
// SetMetas saves metadata for users and posts
|
||||
func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []PostMetaData) (err error) {
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
|
||||
// save posts metas
|
||||
for _, pm := range pmetas {
|
||||
if pm.ReadOnly {
|
||||
errs = multierror.Append(errs, s.SetReadOnly(store.Locator{SiteID: siteID, URL: pm.URL}, true))
|
||||
errs = append(errs, s.SetReadOnly(store.Locator{SiteID: siteID, URL: pm.URL}, true))
|
||||
}
|
||||
}
|
||||
|
||||
// save users metas
|
||||
for _, um := range umetas {
|
||||
if um.Blocked.Status {
|
||||
errs = multierror.Append(errs, s.SetBlock(siteID, um.ID, true, time.Until(um.Blocked.Until)))
|
||||
errs = append(errs, s.SetBlock(siteID, um.ID, true, time.Until(um.Blocked.Until)))
|
||||
}
|
||||
if um.Verified {
|
||||
errs = multierror.Append(errs, s.SetVerified(siteID, um.ID, true))
|
||||
errs = append(errs, s.SetVerified(siteID, um.ID, true))
|
||||
}
|
||||
// this code doesn't delete user details in case they are not set in import but present in DB already
|
||||
if um.Details.Email != "" {
|
||||
req := engine.UserDetailRequest{Locator: store.Locator{SiteID: siteID}, UserID: um.ID, Detail: engine.UserEmail, Update: um.Details.Email}
|
||||
_, err := s.Engine.UserDetail(req)
|
||||
errs = multierror.Append(errs, err)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return errs.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// User gets comment for given userID on siteID
|
||||
@@ -972,15 +973,15 @@ func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.U
|
||||
|
||||
// Close store service
|
||||
func (s *DataStore) Close() error {
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
if s.repliesCache.LoadingCache != nil {
|
||||
errs = multierror.Append(errs, s.repliesCache.Close())
|
||||
errs = append(errs, s.repliesCache.Close())
|
||||
}
|
||||
if s.TitleExtractor != nil {
|
||||
errs = multierror.Append(errs, s.TitleExtractor.Close())
|
||||
errs = append(errs, s.TitleExtractor.Close())
|
||||
}
|
||||
errs = multierror.Append(errs, s.Engine.Close())
|
||||
return errs.ErrorOrNil()
|
||||
errs = append(errs, s.Engine.Close())
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
|
||||
@@ -1011,25 +1012,28 @@ func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
|
||||
|
||||
func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []store.Comment) {
|
||||
res = make([]store.Comment, len(cc))
|
||||
flags := s.newUserFlagCache()
|
||||
for i, c := range cc {
|
||||
res[i] = s.alterComment(c, user)
|
||||
res[i] = s.alterCommentCached(c, user, flags)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) {
|
||||
blocReq := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
|
||||
blocked, bErr := s.Engine.Flag(blocReq)
|
||||
return s.alterCommentCached(c, user, s.newUserFlagCache())
|
||||
}
|
||||
|
||||
// alterCommentCached is alterComment sharing a userFlagCache so that block/verified
|
||||
// lookups for a user repeated across a listing hit the engine only once.
|
||||
func (s *DataStore) alterCommentCached(c store.Comment, user store.User, flags *userFlagCache) (res store.Comment) {
|
||||
// mark user blocked
|
||||
if bErr == nil && blocked {
|
||||
c.User.Blocked = blocked
|
||||
if flags.blocked(c.Locator.SiteID, c.User.ID) {
|
||||
c.User.Blocked = true
|
||||
}
|
||||
|
||||
// set verified status retroactively
|
||||
if !c.User.Blocked {
|
||||
verifReq := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
|
||||
c.User.Verified, _ = s.Engine.Flag(verifReq)
|
||||
c.User.Verified = flags.verified(c.Locator.SiteID, c.User.ID)
|
||||
}
|
||||
|
||||
// hide info from non-admins
|
||||
@@ -1043,6 +1047,51 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co
|
||||
return c
|
||||
}
|
||||
|
||||
// userFlagCache memoises engine block/verified flag lookups by site and user within
|
||||
// a single listing, avoiding two engine.Flag calls per comment for repeated users.
|
||||
type userFlagCache struct {
|
||||
s *DataStore
|
||||
blockedM map[flagKey]bool
|
||||
verifiedM map[flagKey]bool
|
||||
}
|
||||
|
||||
type flagKey struct {
|
||||
siteID string
|
||||
userID string
|
||||
}
|
||||
|
||||
func (s *DataStore) newUserFlagCache() *userFlagCache {
|
||||
return &userFlagCache{s: s, blockedM: map[flagKey]bool{}, verifiedM: map[flagKey]bool{}}
|
||||
}
|
||||
|
||||
func (f *userFlagCache) blocked(siteID, userID string) bool {
|
||||
key := flagKey{siteID: siteID, userID: userID}
|
||||
if v, ok := f.blockedM[key]; ok {
|
||||
return v
|
||||
}
|
||||
v, err := f.s.Engine.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: siteID}, UserID: userID})
|
||||
if err != nil {
|
||||
// don't cache on error so a repeated user is retried, matching the
|
||||
// pre-refactor per-comment behavior; treat this comment as not blocked
|
||||
return false
|
||||
}
|
||||
f.blockedM[key] = v
|
||||
return v
|
||||
}
|
||||
|
||||
func (f *userFlagCache) verified(siteID, userID string) bool {
|
||||
key := flagKey{siteID: siteID, userID: userID}
|
||||
if v, ok := f.verifiedM[key]; ok {
|
||||
return v
|
||||
}
|
||||
v, err := f.s.Engine.Flag(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: siteID}, UserID: userID})
|
||||
if err != nil {
|
||||
return false // don't cache on error, retry on the next comment for this user
|
||||
}
|
||||
f.verifiedM[key] = v
|
||||
return v
|
||||
}
|
||||
|
||||
// prepare vote info for client view
|
||||
func (s *DataStore) prepVotes(c store.Comment, user store.User) store.Comment {
|
||||
c.Vote = 0 // default is "none" (not voted)
|
||||
|
||||
@@ -38,6 +38,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
|
||||
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
|
||||
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
|
||||
}
|
||||
beforeCreate := time.Now()
|
||||
id, err := b.Create(comment)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, id != "", id)
|
||||
@@ -46,7 +47,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
t.Logf("%+v", res)
|
||||
assert.Equal(t, "text", res.Text)
|
||||
assert.True(t, time.Since(res.Timestamp).Seconds() < 1)
|
||||
assert.WithinRange(t, res.Timestamp, beforeCreate, time.Now(), "timestamp set during create")
|
||||
assert.Equal(t, "user", res.User.ID)
|
||||
assert.Equal(t, "name", res.User.Name)
|
||||
assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
|
||||
@@ -218,9 +219,9 @@ func TestService_Put(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestService_SetTitle(t *testing.T) {
|
||||
var titleEnable int32
|
||||
var titleEnable atomic.Int32
|
||||
tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if atomic.LoadInt32(&titleEnable) == 0 {
|
||||
if titleEnable.Load() == 0 {
|
||||
w.WriteHeader(404)
|
||||
}
|
||||
if r.URL.String() == "/post1" {
|
||||
@@ -262,7 +263,7 @@ func TestService_SetTitle(t *testing.T) {
|
||||
|
||||
b.TitleExtractor.cache.Purge()
|
||||
|
||||
atomic.StoreInt32(&titleEnable, 1)
|
||||
titleEnable.Store(1)
|
||||
c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "post1 blah 123", c.PostTitle)
|
||||
@@ -1878,6 +1879,78 @@ func TestService_alterComment(t *testing.T) {
|
||||
assert.Equal(t, engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}, engineMock.FlagCalls()[0].Req)
|
||||
}
|
||||
|
||||
func TestService_alterCommentsFlagCaching(t *testing.T) {
|
||||
t.Run("repeated user looked up once", func(t *testing.T) {
|
||||
engineMock := engine.InterfaceMock{
|
||||
FlagFunc: func(engine.FlagRequest) (bool, error) { return false, nil },
|
||||
}
|
||||
svc := DataStore{Engine: &engineMock}
|
||||
|
||||
comments := make([]store.Comment, 0, 5)
|
||||
for i := range 5 {
|
||||
comments = append(comments, store.Comment{ID: fmt.Sprintf("c%d", i),
|
||||
User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}})
|
||||
}
|
||||
svc.alterComments(comments, store.User{ID: "u1"})
|
||||
|
||||
// one Blocked + one Verified lookup for the single user, not two per comment
|
||||
assert.Equal(t, 2, len(engineMock.FlagCalls()), "5 comments by one user -> 2 flag lookups")
|
||||
})
|
||||
|
||||
t.Run("distinct users looked up per user", func(t *testing.T) {
|
||||
engineMock := engine.InterfaceMock{
|
||||
FlagFunc: func(req engine.FlagRequest) (bool, error) {
|
||||
return req.Flag == engine.Blocked && req.UserID == "blocked", nil // "blocked" user is blocked
|
||||
},
|
||||
}
|
||||
svc := DataStore{Engine: &engineMock}
|
||||
|
||||
comments := []store.Comment{
|
||||
{ID: "c1", User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
{ID: "c2", User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
{ID: "c3", User: store.User{ID: "blocked"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
{ID: "c4", User: store.User{ID: "blocked"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
}
|
||||
res := svc.alterComments(comments, store.User{ID: "admin", Admin: true})
|
||||
|
||||
// u1: Blocked+Verified (2); blocked user: Blocked only, Verified skipped (1) = 3 total
|
||||
assert.Equal(t, 3, len(engineMock.FlagCalls()), "two distinct users -> 3 flag lookups")
|
||||
assert.True(t, res[2].User.Blocked && res[3].User.Blocked, "blocked user marked blocked")
|
||||
assert.False(t, res[0].User.Blocked, "u1 not blocked")
|
||||
})
|
||||
|
||||
t.Run("flag read error is not cached", func(t *testing.T) {
|
||||
var blockedCalls int
|
||||
engineMock := engine.InterfaceMock{
|
||||
FlagFunc: func(req engine.FlagRequest) (bool, error) {
|
||||
if req.Flag == engine.Blocked {
|
||||
blockedCalls++
|
||||
if blockedCalls == 1 {
|
||||
return false, fmt.Errorf("transient flag read error")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
},
|
||||
}
|
||||
svc := DataStore{Engine: &engineMock}
|
||||
|
||||
comments := []store.Comment{
|
||||
{ID: "c0", User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
{ID: "c1", User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
{ID: "c2", User: store.User{ID: "u1"}, Locator: store.Locator{SiteID: "site1"}},
|
||||
}
|
||||
res := svc.alterComments(comments, store.User{ID: "admin", Admin: true})
|
||||
|
||||
// the errored first lookup must not be cached, so the next comment retries and
|
||||
// picks up the real blocked state; once it succeeds the result is cached
|
||||
assert.False(t, res[0].User.Blocked, "errored lookup treated as not blocked")
|
||||
assert.True(t, res[1].User.Blocked, "retry after error picks up blocked state")
|
||||
assert.True(t, res[2].User.Blocked, "successful read is cached")
|
||||
assert.Equal(t, 2, blockedCalls, "blocked retried once after the error, then cached")
|
||||
})
|
||||
}
|
||||
|
||||
func Benchmark_ServiceCreate(b *testing.B) {
|
||||
dbFile := fmt.Sprintf("%s/test-remark42-%d.db", os.TempDir(), rand.Intn(9999999999))
|
||||
defer func() { _ = os.Remove(dbFile) }()
|
||||
|
||||
@@ -44,10 +44,10 @@ func TestTitle_GetTitle(t *testing.T) {
|
||||
func TestTitle_Get(t *testing.T) {
|
||||
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
|
||||
defer ex.Close()
|
||||
var hits int32
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.String() == "/good" {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
hits.Add(1)
|
||||
_, err := w.Write([]byte("<html><title>\n\n blah 123\n</title><body> 2222</body></html>"))
|
||||
assert.NoError(t, err)
|
||||
return
|
||||
@@ -68,7 +68,7 @@ func TestTitle_Get(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "blah 123", r)
|
||||
}
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
|
||||
assert.Equal(t, int32(1), hits.Load())
|
||||
}
|
||||
|
||||
func TestTitle_GetConcurrent(t *testing.T) {
|
||||
@@ -78,10 +78,10 @@ func TestTitle_GetConcurrent(t *testing.T) {
|
||||
}
|
||||
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
|
||||
defer ex.Close()
|
||||
var hits int32
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.String(), "/good") {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
hits.Add(1)
|
||||
_, err := fmt.Fprintf(w, "<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body.String())
|
||||
assert.NoError(t, err)
|
||||
return
|
||||
@@ -100,15 +100,15 @@ func TestTitle_GetConcurrent(t *testing.T) {
|
||||
})
|
||||
}
|
||||
g.Wait()
|
||||
assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
|
||||
assert.Equal(t, int32(100), hits.Load())
|
||||
}
|
||||
|
||||
func TestTitle_GetFailed(t *testing.T) {
|
||||
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second}, []string{"127.0.0.1"})
|
||||
defer ex.Close()
|
||||
var hits int32
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
hits.Add(1)
|
||||
w.WriteHeader(404)
|
||||
}))
|
||||
defer ts.Close()
|
||||
@@ -121,7 +121,7 @@ func TestTitle_GetFailed(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", r)
|
||||
}
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
|
||||
assert.Equal(t, int32(1), hits.Load(), "hit once, errors cached")
|
||||
}
|
||||
|
||||
func TestTitle_DoubleClosed(t *testing.T) {
|
||||
|
||||
@@ -211,9 +211,9 @@ func (t *Tree) limit(limit int, offsetID string) {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if we just exceeded the limit and there are already some nodes in the list,
|
||||
// as otherwise we would have to return the first node with all its replies even if it exceeds the limit.
|
||||
if commentsCount+repliesCount >= limit && len(limitedNodes) > 0 {
|
||||
// stop once adding this subtree would exceed the limit, as long as we already have a node;
|
||||
// a subtree that fits exactly is still included, and the first node is always returned in full.
|
||||
if commentsCount+repliesCount > limit && len(limitedNodes) > 0 {
|
||||
t.countLeft += repliesCount
|
||||
commentsCount = limit // adjust commentsCount to stop checking limit for the next nodes
|
||||
continue
|
||||
|
||||
@@ -152,6 +152,100 @@ func TestTreeSortNodes(t *testing.T) {
|
||||
assert.Equal(t, "1", res.Nodes[0].Comment.ID)
|
||||
}
|
||||
|
||||
func TestMakeTreeLimit(t *testing.T) {
|
||||
loc := store.Locator{URL: "url", SiteID: "site"}
|
||||
ts := func(sec int) time.Time { return time.Date(2017, 12, 25, 19, 0, sec, 0, time.UTC) }
|
||||
|
||||
// tree with four top-level comments and subtree sizes 3, 2, 1, 3 (total 9):
|
||||
// c1 -> c1a, c1b
|
||||
// c2 -> c2a
|
||||
// c3
|
||||
// c4 -> c4a -> c4a1
|
||||
comments := []store.Comment{
|
||||
{Locator: loc, ID: "c1", Timestamp: ts(1)},
|
||||
{Locator: loc, ID: "c1a", ParentID: "c1", Timestamp: ts(11)},
|
||||
{Locator: loc, ID: "c1b", ParentID: "c1", Timestamp: ts(12)},
|
||||
{Locator: loc, ID: "c2", Timestamp: ts(2)},
|
||||
{Locator: loc, ID: "c2a", ParentID: "c2", Timestamp: ts(21)},
|
||||
{Locator: loc, ID: "c3", Timestamp: ts(3)},
|
||||
{Locator: loc, ID: "c4", Timestamp: ts(4)},
|
||||
{Locator: loc, ID: "c4a", ParentID: "c4", Timestamp: ts(41)},
|
||||
{Locator: loc, ID: "c4a1", ParentID: "c4a", Timestamp: ts(42)},
|
||||
}
|
||||
|
||||
nodeIDs := func(nodes []*Node) []string {
|
||||
ids := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
ids = append(ids, n.Comment.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
limit int
|
||||
offsetID string
|
||||
wantNodes []string
|
||||
wantLeft int
|
||||
wantLast string
|
||||
}{
|
||||
{"no limit, no offset returns all", 0, "", []string{"c1", "c2", "c3", "c4"}, 0, ""},
|
||||
{"limit equals first subtree size", 3, "", []string{"c1"}, 6, "c1"},
|
||||
{"limit smaller than first subtree returns it whole", 2, "", []string{"c1"}, 6, "c1"},
|
||||
{"limit between first and second boundary stops after first", 4, "", []string{"c1"}, 6, "c1"},
|
||||
{"limit at exact two-subtree boundary includes both", 5, "", []string{"c1", "c2"}, 4, "c2"},
|
||||
{"limit reaches third subtree exactly", 6, "", []string{"c1", "c2", "c3"}, 3, "c3"},
|
||||
{"limit equal to total returns all", 9, "", []string{"c1", "c2", "c3", "c4"}, 0, "c4"},
|
||||
{"limit larger than total returns all", 100, "", []string{"c1", "c2", "c3", "c4"}, 0, "c4"},
|
||||
{"offset only, no limit slices remainder", 0, "c1", []string{"c2", "c3", "c4"}, 0, ""},
|
||||
{"offset at last node clears result", 0, "c4", []string{}, 0, ""},
|
||||
{"offset at last node with limit clears result", 5, "c4", []string{}, 0, ""},
|
||||
{"offset not found starts from beginning", 0, "missing", []string{"c1", "c2", "c3", "c4"}, 0, ""},
|
||||
{"offset plus limit returns single subtree", 2, "c1", []string{"c2"}, 4, "c2"},
|
||||
{"offset plus limit stops before last subtree", 3, "c2", []string{"c3"}, 3, "c3"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res := MakeTree(comments, "+time", tc.limit, tc.offsetID)
|
||||
assert.Equal(t, tc.wantNodes, nodeIDs(res.Nodes), "top-level nodes")
|
||||
assert.Equal(t, tc.wantLeft, res.CountLeft(), "count left")
|
||||
assert.Equal(t, tc.wantLast, res.LastComment(), "last comment")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountReplies(t *testing.T) {
|
||||
loc := store.Locator{URL: "url", SiteID: "site"}
|
||||
ts := func(sec int) time.Time { return time.Date(2017, 12, 25, 19, 0, sec, 0, time.UTC) }
|
||||
comments := []store.Comment{
|
||||
{Locator: loc, ID: "c1", Timestamp: ts(1)},
|
||||
{Locator: loc, ID: "c1a", ParentID: "c1", Timestamp: ts(11)},
|
||||
{Locator: loc, ID: "c1b", ParentID: "c1", Timestamp: ts(12)},
|
||||
{Locator: loc, ID: "c4", Timestamp: ts(4)},
|
||||
{Locator: loc, ID: "c4a", ParentID: "c4", Timestamp: ts(41)},
|
||||
{Locator: loc, ID: "c4a1", ParentID: "c4a", Timestamp: ts(42)},
|
||||
}
|
||||
res := MakeTree(comments, "+time", 0, "")
|
||||
|
||||
byID := map[string]*Node{}
|
||||
for _, n := range res.Nodes {
|
||||
byID[n.Comment.ID] = n
|
||||
}
|
||||
|
||||
// guard presence and shape first so a regression in MakeTree fails with a clear
|
||||
// assertion instead of a nil-pointer panic on the map lookups below
|
||||
require.Contains(t, byID, "c1")
|
||||
require.Contains(t, byID, "c4")
|
||||
require.Len(t, byID["c1"].Replies, 2)
|
||||
require.Len(t, byID["c4"].Replies, 1)
|
||||
|
||||
assert.Equal(t, 2, countReplies(byID["c1"]), "c1 has two direct replies, no nesting")
|
||||
assert.Equal(t, 2, countReplies(byID["c4"]), "c4 counts nested reply recursively")
|
||||
assert.Equal(t, 1, countReplies(byID["c4"].Replies[0]), "c4a has one nested reply")
|
||||
assert.Equal(t, 0, countReplies(byID["c1"].Replies[0]), "leaf reply has no replies")
|
||||
}
|
||||
|
||||
func BenchmarkTree(b *testing.B) {
|
||||
comments := []store.Comment{}
|
||||
data, err := os.ReadFile("testdata/tree_bench.json")
|
||||
|
||||
|
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 10 KiB |
+6
-8
@@ -7,7 +7,6 @@
|
||||
<meta name="description" content="" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
/* stylelint-disable mavrin/stylelint-declaration-use-css-custom-properties */
|
||||
html {
|
||||
color: #222;
|
||||
font-size: 1em;
|
||||
@@ -182,7 +181,7 @@
|
||||
|
||||
.visuallyhidden {
|
||||
border: 0;
|
||||
clip: rect(0 0 0 0);
|
||||
clip-path: inset(50%);
|
||||
height: 1px;
|
||||
margin: -1px;
|
||||
overflow: hidden;
|
||||
@@ -193,7 +192,7 @@
|
||||
|
||||
.visuallyhidden.focusable:active,
|
||||
.visuallyhidden.focusable:focus {
|
||||
clip: auto;
|
||||
clip-path: none;
|
||||
height: auto;
|
||||
margin: 0;
|
||||
overflow: visible;
|
||||
@@ -254,7 +253,7 @@
|
||||
pre,
|
||||
blockquote {
|
||||
border: 1px solid #999;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid;
|
||||
}
|
||||
|
||||
thead {
|
||||
@@ -263,7 +262,7 @@
|
||||
|
||||
tr,
|
||||
img {
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid;
|
||||
}
|
||||
|
||||
img {
|
||||
@@ -279,7 +278,7 @@
|
||||
|
||||
h2,
|
||||
h3 {
|
||||
page-break-after: avoid;
|
||||
break-after: avoid;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -367,8 +366,7 @@
|
||||
|------------ | -------------|
|
||||
|Content from cell 1 | Content from cell 2|
|
||||
|Content in the first column | Content in the second column|
|
||||
</pre
|
||||
>
|
||||
</pre>
|
||||
</article>
|
||||
|
||||
<aside>
|
||||
@@ -0,0 +1,18 @@
|
||||
// Package webassets holds the files served under /web that the frontend build does not produce:
|
||||
// plain pages and images with no dependency on the bundler's output, embedded into the binary.
|
||||
// A file of the same name in the frontend output, on disk under --web-root or embedded at
|
||||
// app/cmd/web, is served instead, which is how an operator replaces one of these.
|
||||
// Email and error-page templates are a separate set and live in app/templates.
|
||||
package webassets
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
//go:embed assets
|
||||
var embedded embed.FS
|
||||
|
||||
// FS holds the assets, each named by its path under /web. fs.Sub cannot fail for a constant
|
||||
// valid path on an embed.FS, so the error is dropped the same way app/cmd/web's is.
|
||||
var FS, _ = fs.Sub(embedded, "assets")
|
||||
@@ -0,0 +1,78 @@
|
||||
package webassets
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFS_Contents(t *testing.T) {
|
||||
entries, err := fs.ReadDir(FS, ".")
|
||||
require.NoError(t, err)
|
||||
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
assert.False(t, e.IsDir(), "the assets are served flat under /web, %s is a directory", e.Name())
|
||||
info, err := e.Info()
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, info.Size(), "%s is empty", e.Name())
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
assert.Equal(t, []string{"400x400.jpeg", "markdown-help.html", "privacy.html"}, names)
|
||||
}
|
||||
|
||||
// TestFS_ContentShape catches an asset that has been truncated or replaced by something of the
|
||||
// wrong kind, which a size check alone lets through.
|
||||
func TestFS_ContentShape(t *testing.T) {
|
||||
tbl := []struct {
|
||||
name string
|
||||
prefix []byte
|
||||
want string
|
||||
}{
|
||||
{name: "400x400.jpeg", prefix: []byte{0xff, 0xd8, 0xff}},
|
||||
{name: "markdown-help.html", want: "<!DOCTYPE html>"},
|
||||
{name: "privacy.html", want: "<!DOCTYPE html>"},
|
||||
}
|
||||
|
||||
for _, tt := range tbl {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b, err := fs.ReadFile(FS, tt.name)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tt.prefix != nil {
|
||||
require.GreaterOrEqual(t, len(b), len(tt.prefix))
|
||||
assert.Equal(t, tt.prefix, b[:len(tt.prefix)], "not a JPEG")
|
||||
return
|
||||
}
|
||||
assert.True(t, strings.HasPrefix(strings.TrimSpace(string(b)), tt.want), "not an HTML document")
|
||||
assert.Contains(t, string(b), "</html>", "the document is truncated")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFS_RelativeReferencesResolve keeps the pages self-contained: every relative src and href
|
||||
// they use has to name a sibling that ships alongside them, since nothing else supplies one.
|
||||
func TestFS_RelativeReferencesResolve(t *testing.T) {
|
||||
ref := regexp.MustCompile(`(?:src|href)="([^"]+)"`)
|
||||
external := regexp.MustCompile(`^(?:[a-z]+:|//|#|mailto:)`)
|
||||
|
||||
for _, page := range []string{"markdown-help.html", "privacy.html"} {
|
||||
t.Run(page, func(t *testing.T) {
|
||||
b, err := fs.ReadFile(FS, page)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, m := range ref.FindAllStringSubmatch(string(b), -1) {
|
||||
target := m[1]
|
||||
if external.MatchString(target) {
|
||||
continue
|
||||
}
|
||||
_, err := fs.Stat(FS, target)
|
||||
assert.NoError(t, err, "%s references %q, which ships nowhere", page, target)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+24
-29
@@ -7,31 +7,30 @@ require (
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/didip/tollbooth/v8 v8.0.1
|
||||
github.com/go-pkgz/auth/v2 v2.1.5
|
||||
github.com/go-pkgz/jrpc v0.4.0
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0
|
||||
github.com/go-pkgz/lgr v0.12.3
|
||||
github.com/go-pkgz/notify v1.3.0
|
||||
github.com/go-pkgz/auth/v2 v2.2.0
|
||||
github.com/go-pkgz/jrpc v0.4.2
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0
|
||||
github.com/go-pkgz/lgr v0.12.4
|
||||
github.com/go-pkgz/notify v1.4.0
|
||||
github.com/go-pkgz/repeater/v2 v2.2.0
|
||||
github.com/go-pkgz/rest v1.22.0
|
||||
github.com/go-pkgz/routegroup v1.6.0
|
||||
github.com/go-pkgz/syncs v1.3.2
|
||||
github.com/go-pkgz/rest v1.24.0
|
||||
github.com/go-pkgz/routegroup v1.6.1
|
||||
github.com/go-pkgz/syncs v1.3.3
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/feeds v1.2.0
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/kyokomi/emoji/v2 v2.2.13
|
||||
github.com/kyokomi/emoji/v2 v2.2.14
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/rs/xid v1.6.0
|
||||
github.com/russross/blackfriday/v2 v2.1.0
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
go.etcd.io/bbolt v1.5.0
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/image v0.43.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/crypto v0.55.0
|
||||
golang.org/x/image v0.45.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
)
|
||||
|
||||
@@ -40,32 +39,28 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dghubble/oauth1 v0.7.3 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 // indirect
|
||||
github.com/go-oauth2/oauth2/v4 v4.5.4 // indirect
|
||||
github.com/go-pkgz/email v0.6.0 // indirect
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0 // indirect
|
||||
github.com/go-pkgz/repeater v1.2.0 // indirect
|
||||
github.com/go-pkgz/email v0.8.0 // indirect
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/klauspost/compress v1.18.7 // indirect
|
||||
github.com/montanaflynn/stats v0.9.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/redis/go-redis/v9 v9.21.0 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/montanaflynn/stats v0.12.4 // indirect
|
||||
github.com/redis/go-redis/v9 v9.22.0 // indirect
|
||||
github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d // indirect
|
||||
github.com/slack-go/slack v0.27.0 // indirect
|
||||
github.com/slack-go/slack v0.29.0 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
)
|
||||
|
||||
+52
-66
@@ -12,10 +12,8 @@ github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r
|
||||
github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
|
||||
github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
@@ -34,38 +32,36 @@ github.com/dghubble/oauth1 v0.7.3 h1:EkEM/zMDMp3zOsX2DC/ZQ2vnEX3ELK0/l9kb+vs4ptE
|
||||
github.com/dghubble/oauth1 v0.7.3/go.mod h1:oxTe+az9NSMIucDPDCCtzJGsPhciJV33xocHfcR2sVY=
|
||||
github.com/didip/tollbooth/v8 v8.0.1 h1:VAAapTo1t4Bn6bbpcHjuovwoa9u3JH++wgjbpWv+rB8=
|
||||
github.com/didip/tollbooth/v8 v8.0.1/go.mod h1:oEd9l+ep373d7DmvKLc0a5gasPOev2mTewi6KPQBGJ4=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8=
|
||||
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
|
||||
github.com/go-oauth2/oauth2/v4 v4.5.4 h1:YjI0tmGW8oxVhn9QSBIxlr641QugWrJY5UWa6XmLcW0=
|
||||
github.com/go-oauth2/oauth2/v4 v4.5.4/go.mod h1:BXiOY+QZtZy2ewbsGk2B5P8TWmtz/Rf7ES5ZttQFxfQ=
|
||||
github.com/go-pkgz/auth/v2 v2.1.5 h1:CFL7XxRMNPga0S0YCnAnlvO61OHHEYvVEGrIZXuA98Y=
|
||||
github.com/go-pkgz/auth/v2 v2.1.5/go.mod h1:IvxxhJIrwd1hKqFwQgBF9i+sMTmGfzAw66wmhw1zfJc=
|
||||
github.com/go-pkgz/email v0.6.0 h1:snZnXldjeF4PgKSjnx9Fa25mtOgFpAOEeWvnQvrxjLE=
|
||||
github.com/go-pkgz/email v0.6.0/go.mod h1:+wgi4x7S33IuCzfcCM5euN0GwQG6XvO/PBLxrNffYLI=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0 h1:s05P851/O6QJ6Mc+7o2bh9aGtD3romB1SxDTXifdoqc=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0/go.mod h1:6pVgNleydKPj0J2/mzrI02/RDo4ivKx5v2XlNmIjhjo=
|
||||
github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY=
|
||||
github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo=
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0 h1:gTwXpiJBhQeA1rXuqkRuLcV79uATFna8CckH8ZBBrH0=
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0/go.mod h1:yxJHOn+IbQBQHxUqkCtMrbGjIfdYcsBAZcVCBaL1Va8=
|
||||
github.com/go-pkgz/lgr v0.12.3 h1:QDug7kRkEsuQtruT9fNF5PVT2kZUqCDPc4GmsgS3fP8=
|
||||
github.com/go-pkgz/lgr v0.12.3/go.mod h1:lpCDgVvCIxBHZp8+sGCj9MPctIzKZyZ3QdE19ddqd54=
|
||||
github.com/go-pkgz/notify v1.3.0 h1:YxF/ThEoCetdcoghWdyeqaBpCkZ8mvyve7HXbCAOzYU=
|
||||
github.com/go-pkgz/notify v1.3.0/go.mod h1:qdfi5OsViKlIFPryIOaINHTOtS9GFhOYXPqJmAMlaGU=
|
||||
github.com/go-pkgz/repeater v1.2.0 h1:oJFvjyKdTDd5RCzpzxlzYIZFFj6Zfl17rE1aUfu6UjQ=
|
||||
github.com/go-pkgz/repeater v1.2.0/go.mod h1:vypP6xamA53MFmafnGUucqOmALKk36xgKu2hSG73LHM=
|
||||
github.com/go-pkgz/auth/v2 v2.2.0 h1:vQO+GTFDjAaBNSdcLLLr3Xibka67GZPtkRRItVVS4ow=
|
||||
github.com/go-pkgz/auth/v2 v2.2.0/go.mod h1:iZx2JiGZ8Aef+wM0BPLMQY8aur4fLE0uyPhFRO9dYQ4=
|
||||
github.com/go-pkgz/email v0.8.0 h1:6+Tgjfj7zFccFCPmURV2spKDXDb7aX/iWXBY2hBa9ww=
|
||||
github.com/go-pkgz/email v0.8.0/go.mod h1:+wgi4x7S33IuCzfcCM5euN0GwQG6XvO/PBLxrNffYLI=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1 h1:ryHiSI5gBE8aJ0Jt90VFMKZxe/cosf2dZPDcUTMaHNg=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1/go.mod h1:peJAuIDjP76Uuc9NK55ljQlBtwwmJDvx4CnMyUcsP40=
|
||||
github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU=
|
||||
github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ=
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0 h1:JAGUHRQPon658XimxIUwaqPgOPfIKa850qO8jpFJchc=
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0/go.mod h1:UUo4cgD6oTPooBuUslVaWqOYZAGnh/91SoaB5DBWC8s=
|
||||
github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok=
|
||||
github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8=
|
||||
github.com/go-pkgz/notify v1.4.0 h1:4pP7UGdYqFO7e7V3OsQStYF006CO0cCh1ahdawt6l18=
|
||||
github.com/go-pkgz/notify v1.4.0/go.mod h1:UFpL9ZvCYnLBEjeay++3afh8GceZ6qT8wj5lqfSR9U4=
|
||||
github.com/go-pkgz/repeater/v2 v2.2.0 h1:8nZR/NaknmLfx2YMHbr78u9OL4Xj+8+romm9dz4FpMg=
|
||||
github.com/go-pkgz/repeater/v2 v2.2.0/go.mod h1:RgX5vUbLKq7PV82QUDP5pFbQS1os4Z+U9XzKymK23A8=
|
||||
github.com/go-pkgz/rest v1.22.0 h1:d3XFKlmAGBiU9MQER9/n46iXpyUr8IQUtfjU8JlqkkY=
|
||||
github.com/go-pkgz/rest v1.22.0/go.mod h1:+AHzjHazq7Z3Tk/kRWOhbbAz/YZlUV40feC1Hf4NtbE=
|
||||
github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4=
|
||||
github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/go-pkgz/syncs v1.3.2 h1:gmioASlJNy3gNosPlgvWOM2QP0Hdjzn2u+/sUShgd8E=
|
||||
github.com/go-pkgz/syncs v1.3.2/go.mod h1:qjgzpp7OpuhDf7BWsW/FHCu9DLjE32NPy6/vXAXT/Cw=
|
||||
github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI=
|
||||
github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc=
|
||||
github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q=
|
||||
github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/go-pkgz/syncs v1.3.3 h1:fFRK+eqCIFddxEiDi6ob5oQh3/NuNkjuwWQJzZ0b9lU=
|
||||
github.com/go-pkgz/syncs v1.3.3/go.mod h1:lAp+w+qbRm6+pwINcfc9BVK+4F4NEoERYditsF1uENA=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
@@ -86,11 +82,6 @@ github.com/gorilla/feeds v1.2.0 h1:O6pBiXJ5JHhPvqy53NsjKOThq+dNFm8+DFrxBEdzSCc=
|
||||
github.com/gorilla/feeds v1.2.0/go.mod h1:WMib8uJP3BbY+X8Szd1rA5Pzhdfh+HCCAYT2z7Fza6Y=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
@@ -101,26 +92,24 @@ github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bB
|
||||
github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U=
|
||||
github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
|
||||
github.com/kyokomi/emoji/v2 v2.2.14 h1:YOF6VL52613M0Qr9v4puJDD9QQPmyyjXedDDlrGzH80=
|
||||
github.com/kyokomi/emoji/v2 v2.2.14/go.mod h1:1AnYl9IgmJZXKd5m1PEijyyUw85SqYsuAr8lpU/s+9s=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/montanaflynn/stats v0.9.0 h1:tsBJ0RXwph9BmAuFoCmqGv6e8xa0MENQ8m0ptKq29mQ=
|
||||
github.com/montanaflynn/stats v0.9.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/montanaflynn/stats v0.12.4 h1:amtNRsti20yIhcrkfUJGwoYqBR82jKQFE8SNNYVgGn0=
|
||||
github.com/montanaflynn/stats v0.12.4/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E=
|
||||
github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0=
|
||||
github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d h1:l3+2LWCbVxn5itfvXAfH9n4YL9jh8l1g5zcncbIc1cs=
|
||||
@@ -133,20 +122,20 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
|
||||
github.com/slack-go/slack v0.27.0 h1:VWOpUzOK6UAPCCQlFxl79jhv8a/b+GOSJMnWziDJ8B8=
|
||||
github.com/slack-go/slack v0.27.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4=
|
||||
github.com/slack-go/slack v0.29.0 h1:ohhMNgp9DmPKiLhH/pNZV4NxhOXKgNy0SH8FzVHNerI=
|
||||
github.com/slack-go/slack v0.29.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4=
|
||||
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
|
||||
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
|
||||
github.com/tidwall/buntdb v1.3.2 h1:qd+IpdEGs0pZci37G4jF51+fSKlkuUTMXuHhXL1AkKg=
|
||||
github.com/tidwall/buntdb v1.3.2/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU=
|
||||
github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc=
|
||||
github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
|
||||
github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
|
||||
github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM=
|
||||
@@ -194,45 +183,42 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
|
||||
golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
+43
-5
@@ -38,7 +38,7 @@ if isMatch, _ := re.MatchString(`Something to match`); isMatch {
|
||||
}
|
||||
```
|
||||
|
||||
The only error that the `*Match*` methods *should* return is a Timeout if you set the `re.MatchTimeout` field. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so:
|
||||
The `*Match*` methods can return a timeout error if you set the `re.MatchTimeout` field, or `ErrBacktrackingStackLimit` if a match exceeds its configured backtracking stack size. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so:
|
||||
|
||||
```go
|
||||
if m, _ := re.FindStringMatch(`Something to match`); m != nil {
|
||||
@@ -92,6 +92,17 @@ notEmoji := regexp2.MustCompile(`\P{Emoji}+`)
|
||||
|
||||
Valid property names and aliases come from Unicode 17.0.0 [`PropertyAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyAliases.txt). Valid property values and aliases come from Unicode 17.0.0 [`PropertyValueAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyValueAliases.txt). The generated tables use Unicode 17.0.0 data from [`DerivedCoreProperties.txt`](https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt), [`emoji/emoji-data.txt`](https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt), [`auxiliary/GraphemeBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/GraphemeBreakProperty.txt), [`auxiliary/WordBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/WordBreakProperty.txt), and [`auxiliary/SentenceBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/SentenceBreakProperty.txt) for the package-local properties whose data changes more frequently than the Go standard library tables.
|
||||
|
||||
## Additional Perl and PCRE syntax
|
||||
|
||||
The default mode supports the following syntax:
|
||||
|
||||
* `\Q...\E` quotes every character between `\Q` and `\E`. If `\E` is omitted, quoting continues to the end of the pattern. This also works inside character classes.
|
||||
* `\R` matches one Unicode newline sequence: CRLF as a single sequence, or LF, VT, FF, CR, NEL, line separator, or paragraph separator.
|
||||
* `\X` atomically matches one Unicode 17.0.0 extended grapheme cluster. This includes combining sequences, Hangul syllables, regional-indicator pairs, emoji ZWJ sequences, and Indic conjuncts.
|
||||
* A `+` after a quantifier makes it possessive: `*+`, `++`, `?+`, and `{m,n}+`. For example, `a*+` is equivalent to `(?>a*)` and will not give characters back when the remainder of the pattern fails.
|
||||
|
||||
`RE2` mode also supports `\Q...\E` and possessive quantifiers, but keeps `\R` and `\X` as literal `R` and `X` identity escapes. `ECMAScript` mode does not enable any of this syntax: `\Q`, `\E`, `\R`, and `\X` retain ECMAScript identity-escape behavior, and possessive quantifiers remain invalid.
|
||||
|
||||
## `regexp` compatibility adapter
|
||||
|
||||
The `github.com/dlclark/regexp2/v2/compat` package provides an adapter for callers that want the same `Find*` and `Match*` method signatures as the standard library's `regexp.Regexp`, while still using the `regexp2` engine.
|
||||
@@ -128,7 +139,7 @@ func findWords(re compat.Matcher, input string) []string {
|
||||
}
|
||||
```
|
||||
|
||||
Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout. Use the main `regexp2` APIs directly when you need to handle timeouts as errors.
|
||||
Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout or `ErrBacktrackingStackLimit`. Use the main `regexp2` APIs directly when you need to handle match errors directly.
|
||||
|
||||
## Compile options
|
||||
|
||||
@@ -146,6 +157,7 @@ Performance tuning options override the default cache settings:
|
||||
```go
|
||||
re := regexp2.MustCompile(`Your pattern`,
|
||||
regexp2.IgnoreCase,
|
||||
regexp2.OptionMaxBacktrackingStackSize(200000),
|
||||
regexp2.OptionMaxCachedRuneBufferLength(64*1024),
|
||||
regexp2.OptionMaxCachedReplacerDataEntries(8),
|
||||
)
|
||||
@@ -164,23 +176,30 @@ The defaults are intentionally bounded:
|
||||
| `OptionMaintainCaptureOrder()` | false | Parser capture-slot assignment for mixed named and unnamed captures. | None at match time. This changes compile-time capture numbering only. | Keeps named and unnamed captures in pattern order instead of appending named captures after unnamed captures. This can change numeric backreference meaning, so it is caller-controlled rather than an inline regex option. |
|
||||
| `OptionDebug()` | false | Compile dumps and runner tracing. | Debug output volume only. | Useful for diagnostics, but it can produce noisy output and slower traced matching. |
|
||||
| `OptionIsCodeGen()` | false | Compile-time find-optimization analysis for [`regexp2cg`](https://github.com/dlclark/regexp2cg). | Per compiled regexp, during `Compile` or `MustCompile`. | Enables more expensive analysis intended for generated engines. Do not use it for normal interpreter execution; the interpreter defaults intentionally avoid this extra compile-time cost. |
|
||||
| `OptionMaxBacktrackingStackSize(n)` | 100,000 | The interpreter's per-match backtracking stack. | Per pooled runner. The initial allocation and subsequent growth are capped at the configured number of integer slots; the runner pool may retain stacks at their high-water size for reuse. | Lowering this bounds backtracking memory more tightly but may reject complex matches sooner with `ErrBacktrackingStackLimit`. Raising it permits deeper backtracking and increases possible memory use. A negative value disables the limit. |
|
||||
| `OptionMaxCachedRuneBufferLength(n)` | 256K runes | String APIs that run through pooled runners, such as `MatchString` and replacement-pattern `Replace`, when converting input strings to the engine's internal `[]rune` representation. | Process-wide shared `sync.Pool` retention by size class. This does not grow per compiled regexp or per input string; the practical working set follows recent and concurrent use across all regexps and can be dropped by GC. | Raising this lets calls use larger pooled rune buffers and can reduce allocations for repeated matches against large strings. Lowering it prevents larger buffers from being borrowed or returned, so large inputs allocate directly. |
|
||||
| `OptionMaxCachedReplaceBufferLength(n)` | 256 KB | Replacement-pattern `Replace` calls that build output through a shared byte buffer. | Process-wide shared `sync.Pool` retention by size class after replacement-pattern `Replace` runs. It does not grow from evaluator-based `ReplaceFunc` output and is shared across compiled regexps. | Raising this lets larger replacement outputs use pooled buffers and can reduce allocations. Lowering it prevents larger output buffers from being retained, so large replacements allocate directly. |
|
||||
| `OptionMaxCachedReplacerDataEntries(n)` | `16` | `Replace` with replacement pattern strings, after the replacement pattern is parsed into reusable replacement data. | Per compiled regexp. The cache grows as distinct cacheable replacement strings are used with `Replace`, up to this entry count. | Raising this helps when a single compiled regexp is used with many recurring replacement patterns. It increases per-regexp cache memory and lock-protected cache bookkeeping. Setting it to `0` disables this cache. |
|
||||
| `OptionMaxCachedReplacerDataBytes(n)` | 4 KB | The parsed replacement-pattern cache. Replacement strings longer than this are parsed for the call but not retained. | Per compiled regexp, combined with `OptionMaxCachedReplacerDataEntries`. Only replacement strings whose source text is at or below this size can add parsed data to the cache. | Raising this helps if large replacement patterns are reused. It can retain more memory per cached replacement. Lowering it avoids keeping unusual large replacement patterns around. |
|
||||
| `OptionDisableCharClassASCIIBitmap()` | false | Compile-time preparation of character classes and first-character prefix sets. By default, character classes with ASCII membership get a small bitmap used by `CharIn`. | Per compiled regexp, during `Compile` or `MustCompile`. Each eligible character class can hold one small bitmap; this does not scale with match concurrency or input size. | Leaving this false speeds up ASCII-heavy character class checks at the cost of a small amount of per-char-class memory and compile-time work. Setting to true can reduce memory for large numbers of compiled char classes in regexps, but ASCII character class matching may be slower. |
|
||||
|
||||
For `OptionMaxBacktrackingStackSize`, set `n` to a negative value to allow unbounded stack growth. Setting it to `0` permits no backtracking stack entries, so most interpreted matches will return `ErrBacktrackingStackLimit`.
|
||||
|
||||
For pooled buffer cache options, set `n` to `0` to disable pooling, or `-1` to allow all built-in size classes. The rune buffer classes are 1K, 4K, 16K, 64K, and 256K runes. The replacement byte buffer classes are 4 KB, 16 KB, 64 KB, 256 KB, and 1 MB. By default the 1 MB pool is unused. For replacement data byte-size cache options, `-1` means unbounded. For entry-count cache options, set `n` to `0` to disable the cache.
|
||||
|
||||
## Compare `regexp` and `regexp2`
|
||||
| Category | regexp | regexp2 |
|
||||
| --- | --- | --- |
|
||||
| Catastrophic backtracking possible | no, constant execution time guarantees | yes, if your pattern is at risk you can use the `re.MatchTimeout` field |
|
||||
| Catastrophic backtracking possible | no, constant execution time guarantees | yes; backtracking stack growth is bounded by default, and `re.MatchTimeout` can also bound match duration |
|
||||
| Python-style capture groups `(?P<name>re)` | yes | no (yes in RE2 compat mode) |
|
||||
| .NET-style capture groups `(?<name>re)` or `(?'name're)` | yes | yes |
|
||||
| comments `(?#comment)` | no | yes |
|
||||
| branch numbering reset `(?\|a\|b)` | no | no |
|
||||
| possessive match `(?>re)` | no | yes |
|
||||
| atomic group `(?>re)` | no | yes |
|
||||
| possessive quantifiers `*+`, `++`, `?+`, `{m,n}+` | no | yes |
|
||||
| literal quoting `\Q...\E` | yes | yes |
|
||||
| Unicode newline sequence `\R` | no | yes (default mode only) |
|
||||
| extended grapheme cluster `\X` | no | yes (default mode only) |
|
||||
| positive lookahead `(?=re)` | no | yes |
|
||||
| negative lookahead `(?!re)` | no | yes |
|
||||
| positive lookbehind `(?<=re)` | no | yes |
|
||||
@@ -199,6 +218,8 @@ The default behavior of `regexp2` is to match the .NET regexp engine, however th
|
||||
* change singleline behavior for `$` to only match end of string (like RE2) (see [#24](https://github.com/dlclark/regexp2/issues/24))
|
||||
* change the character classes `\d` `\s` and `\w` to match the same characters as RE2. NOTE: if you also use the `ECMAScript` option then this will change the `\s` character class to match ECMAScript instead of RE2. ECMAScript allows more whitespace characters in `\s` than RE2 (but still fewer than the the default behavior).
|
||||
* allow character escape sequences to have defaults. For example, by default `\_` isn't a known character escape and will fail to compile, but in RE2 mode it will match the literal character `_`
|
||||
* support RE2-style literal quoting with `\Q...\E`
|
||||
* support possessive quantifiers (`*+`, `++`, `?+`, and `{m,n}+`) as a regexp2 extension
|
||||
|
||||
```go
|
||||
re := regexp2.MustCompile(`Your RE2-compatible pattern`, regexp2.RE2)
|
||||
@@ -212,7 +233,22 @@ This feature is a work in progress and I'm open to ideas for more things to put
|
||||
## Catastrophic Backtracking and Timeouts
|
||||
|
||||
`regexp2` supports features that can lead to catastrophic backtracking.
|
||||
`Regexp.MatchTimeout` can be set to to limit the impact of such behavior; the
|
||||
Each compiled regexp limits its per-match backtracking stack to 100,000
|
||||
slots by default. If a match would exceed that limit, it stops and returns
|
||||
`ErrBacktrackingStackLimit`. Callers can identify it with
|
||||
`errors.Is(err, regexp2.ErrBacktrackingStackLimit)`. The limit can be changed at
|
||||
compile time; a negative value restores the previous unbounded behavior:
|
||||
|
||||
```go
|
||||
re := regexp2.MustCompile(pattern, regexp2.OptionMaxBacktrackingStackSize(200000))
|
||||
// regexp2.OptionMaxBacktrackingStackSize(-1) disables the limit.
|
||||
```
|
||||
|
||||
This limit bounds the interpreter's backtracking stack, not total match time or
|
||||
all memory used by a match. Literal empty expressions repeated any number of
|
||||
times are optimized away and do not consume backtracking stack space.
|
||||
|
||||
`Regexp.MatchTimeout` can be set to limit the impact of such behavior; the
|
||||
match will fail with an error after approximately MatchTimeout. No timeout
|
||||
checks are done by default.
|
||||
|
||||
@@ -276,6 +312,8 @@ This flag should not be treated as compatibility with C#'s `RegexOptions.ECMAScr
|
||||
|
||||
Additionally a Unicode mode is provided which allows parsing of `\u{CodePoint}` syntax only when both `ECMAScript` and `Unicode` are provided.
|
||||
|
||||
Perl/PCRE extensions `\Q...\E`, `\R`, `\X`, and possessive quantifiers are intentionally not enabled in this mode. The letter escapes retain the engine's existing ECMAScript identity-escape behavior.
|
||||
|
||||
## Potential bugs
|
||||
I've run a battery of tests against regexp2 from various sources and found the debug output matches the .NET engine, but .NET and Go handle strings very differently. I've attempted to handle these differences, but most of my testing deals with basic ASCII with a little bit of multi-byte Unicode. There's a chance that there are bugs in the string handling related to character sets with supplementary Unicode chars. Right-to-Left support is coded, but not well tested either.
|
||||
|
||||
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
package regexp2
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/dlclark/regexp2/v2/helpers"
|
||||
)
|
||||
|
||||
// decodedInput is the engine's rune view of a string. When the pattern cannot
|
||||
// look behind a candidate start, the slice may begin at that candidate instead
|
||||
// of byte 0.
|
||||
type decodedInput struct {
|
||||
runes []rune
|
||||
pooled *[]rune
|
||||
runeStart int // index in runes of the requested startAt; -1 if not a rune boundary
|
||||
runeOffset int // original-string rune index of runes[0]
|
||||
byteOffset int // original-string byte index of runes[0]
|
||||
}
|
||||
|
||||
// decodeFrom is the first original-string byte that must be decoded. 0 means
|
||||
// the whole string; a later index is only used when slicing is safe.
|
||||
func (re *Regexp) decodeFrom(s string, startAt int) int {
|
||||
if startAt <= 0 || re.RightToLeft() {
|
||||
return 0
|
||||
}
|
||||
need := re.decodeLeftContextRunes()
|
||||
if need < 0 {
|
||||
return 0
|
||||
}
|
||||
if need > 0 {
|
||||
return prevRuneByte(s, startAt)
|
||||
}
|
||||
return startAt
|
||||
}
|
||||
|
||||
func (re *Regexp) decodeLeftContextRunes() int {
|
||||
if re.code != nil {
|
||||
return re.code.LeftContextRunes
|
||||
}
|
||||
return re.leftContextRunes
|
||||
}
|
||||
|
||||
// decodeString converts s to []rune for the MatchString startAt<=0 path.
|
||||
// Keep this as close as possible to a single UTF-8 walk so 4–30 byte
|
||||
// matches stay cheap.
|
||||
func decodeString(s string, maxCachedLength int) ([]rune, *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), maxCachedLength)
|
||||
if len(s) >= helpers.ASCIIScanMin {
|
||||
n, _ := helpers.DecodeString(s, buf)
|
||||
return buf[:n], pooled
|
||||
}
|
||||
n := 0
|
||||
for _, ch := range s {
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return buf[:n], pooled
|
||||
}
|
||||
|
||||
func (re *Regexp) decodeStringInput(s string, startAt int, pooled bool) decodedInput {
|
||||
maxCachedLength := 0
|
||||
if pooled {
|
||||
maxCachedLength = re.optimizations.MaxCachedRuneBufferLength
|
||||
}
|
||||
return decodeInput(s, startAt, re.decodeFrom(s, startAt), maxCachedLength, true)
|
||||
}
|
||||
|
||||
// decodeInput converts s[decodeFrom:] to runes. startAt is a byte index in s.
|
||||
// If startAt < 0 or is not a rune boundary, runeStart is -1. needOffsets walks
|
||||
// the skipped prefix to fill runeOffset; MatchString does not need that field.
|
||||
func decodeInput(s string, startAt, decodeFrom, maxCachedLength int, needOffsets bool) decodedInput {
|
||||
if decodeFrom < 0 {
|
||||
decodeFrom = 0
|
||||
}
|
||||
if decodeFrom > len(s) {
|
||||
decodeFrom = len(s)
|
||||
}
|
||||
|
||||
substr := s[decodeFrom:]
|
||||
buf, pooledBuffer := pooledRuneBuffers.get(len(substr), maxCachedLength)
|
||||
n, ascii := helpers.DecodeString(substr, buf)
|
||||
runes := buf[:n]
|
||||
|
||||
out := decodedInput{
|
||||
runes: runes,
|
||||
pooled: pooledBuffer,
|
||||
byteOffset: decodeFrom,
|
||||
runeStart: startRuneIndex(s, startAt, decodeFrom, n, ascii),
|
||||
}
|
||||
|
||||
if !needOffsets {
|
||||
return out
|
||||
}
|
||||
|
||||
// Byte index equals rune index only when the skipped prefix is also ASCII.
|
||||
if ascii && (decodeFrom == 0 || helpers.IsASCII(s[:decodeFrom])) {
|
||||
out.runeOffset = decodeFrom
|
||||
return out
|
||||
}
|
||||
if decodeFrom > 0 {
|
||||
out.runeOffset = utf8.RuneCountInString(s[:decodeFrom])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func startRuneIndex(s string, startAt, decodeFrom, runeCount int, ascii bool) int {
|
||||
if startAt < 0 {
|
||||
return -1
|
||||
}
|
||||
if startAt < decodeFrom || startAt > len(s) {
|
||||
return -1
|
||||
}
|
||||
if startAt == decodeFrom {
|
||||
return 0
|
||||
}
|
||||
if startAt == len(s) {
|
||||
return runeCount
|
||||
}
|
||||
if ascii {
|
||||
return startAt - decodeFrom
|
||||
}
|
||||
rel := startAt - decodeFrom
|
||||
i := 0
|
||||
for strIdx := range s[decodeFrom:] {
|
||||
if strIdx == rel {
|
||||
return i
|
||||
}
|
||||
if strIdx > rel {
|
||||
return -1
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func prevRuneByte(s string, byteIndex int) int {
|
||||
if byteIndex <= 0 {
|
||||
return 0
|
||||
}
|
||||
_, size := utf8.DecodeLastRuneInString(s[:byteIndex])
|
||||
if size <= 0 {
|
||||
return byteIndex - 1
|
||||
}
|
||||
return byteIndex - size
|
||||
}
|
||||
|
||||
func (d decodedInput) release() {
|
||||
if d.pooled != nil {
|
||||
*d.pooled = d.runes
|
||||
pooledRuneBuffers.put(d.pooled)
|
||||
}
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const asciiHighBits = 0x8080808080808080
|
||||
|
||||
// ASCIIScanMin is the input length where a separate ASCII probe plus a tight
|
||||
// copy beats a single UTF-8 range loop. Short matches stay on the one-pass path.
|
||||
const ASCIIScanMin = 64
|
||||
|
||||
// ASCIISearchMin is the haystack length where stdlib/SIMD string search beats
|
||||
// a scalar first-rune scan. Below this, needle setup and unaligned
|
||||
// bytes.Index retries dominate.
|
||||
const ASCIISearchMin = 32
|
||||
|
||||
// IsASCII reports whether s contains only bytes below utf8.RuneSelf.
|
||||
func IsASCII(s string) bool {
|
||||
n := len(s)
|
||||
if n == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
b := unsafe.Slice(unsafe.StringData(s), n)
|
||||
i := 0
|
||||
for ; i < n && uintptr(unsafe.Pointer(&b[i]))&7 != 0; i++ {
|
||||
if b[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for ; i+8 <= n; i += 8 {
|
||||
v := *(*uint64)(unsafe.Pointer(&b[i]))
|
||||
if v&asciiHighBits != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for ; i < n; i++ {
|
||||
if b[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// IsASCIIRunes reports whether every rune in in is an ASCII code point.
|
||||
func IsASCIIRunes(in []rune) bool {
|
||||
for _, ch := range in {
|
||||
if ch >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// DecodeString writes the runes of s into buf, which must have length >= len(s).
|
||||
// It returns the rune count and whether s was all ASCII.
|
||||
func DecodeString(s string, buf []rune) (n int, ascii bool) {
|
||||
if len(s) >= ASCIIScanMin && IsASCII(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
buf[i] = rune(s[i])
|
||||
}
|
||||
return len(s), true
|
||||
}
|
||||
ascii = true
|
||||
for _, ch := range s {
|
||||
if ch >= utf8.RuneSelf {
|
||||
ascii = false
|
||||
}
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return n, ascii
|
||||
}
|
||||
+107
-49
@@ -11,11 +11,30 @@ import (
|
||||
)
|
||||
|
||||
func IndexOfAny(in []rune, find []rune) int {
|
||||
// special case
|
||||
if len(find) == 0 {
|
||||
switch len(find) {
|
||||
case 0:
|
||||
return -1
|
||||
case 1:
|
||||
return IndexOfAny1(in, find[0])
|
||||
case 2:
|
||||
return IndexOfAny2(in, find[0], find[1])
|
||||
case 3:
|
||||
return IndexOfAny3(in, find[0], find[1], find[2])
|
||||
}
|
||||
|
||||
if IsASCIIRunes(find) {
|
||||
var bits [2]uint64
|
||||
for _, c := range find {
|
||||
bits[c>>6] |= 1 << (c & 63)
|
||||
}
|
||||
for i, c := range in {
|
||||
if uint32(c) < 128 && bits[c>>6]&(1<<(c&63)) != 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
// naive version
|
||||
|
||||
for i, c := range in {
|
||||
if slices.Contains(find, c) {
|
||||
return i
|
||||
@@ -25,8 +44,10 @@ func IndexOfAny(in []rune, find []rune) int {
|
||||
}
|
||||
|
||||
func IndexOfAny1(in []rune, find rune) int {
|
||||
//TODO: bytes optimization?
|
||||
return slices.Index(in, find)
|
||||
if len(in) < ASCIISearchMin {
|
||||
return slices.Index(in, find)
|
||||
}
|
||||
return indexOfRuneBytes(runeSliceBytes(in), find)
|
||||
}
|
||||
|
||||
func IndexOfAny2(in []rune, find1, find2 rune) int {
|
||||
@@ -125,28 +146,34 @@ func IndexFunc(in []rune, f func(ch rune) bool) int {
|
||||
}
|
||||
|
||||
func IndexOfAnyExceptInSet(in []rune, set syntax.CharSet) int {
|
||||
//TODO: this
|
||||
panic("not implemented")
|
||||
for i, c := range in {
|
||||
if !set.CharIn(c) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func LastIndexOf(in []rune, find []rune) int {
|
||||
end := len(in) - len(find)
|
||||
first := find[0]
|
||||
lastOffset := len(find) - 1
|
||||
last := find[lastOffset]
|
||||
for i := end; i >= 0; i-- {
|
||||
//TODO: check 2 chars needed?
|
||||
// match start and end...check the middle
|
||||
if in[i] == first && in[i+lastOffset] == last {
|
||||
// found our first char
|
||||
// check if the rest are equal
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
if len(find) == 0 {
|
||||
return len(in)
|
||||
}
|
||||
if len(in) < len(find) {
|
||||
return -1
|
||||
}
|
||||
haystack := runeSliceBytes(in)
|
||||
needle := runeSliceBytes(find)
|
||||
end := len(haystack)
|
||||
for end >= len(needle) {
|
||||
idx := bytes.LastIndex(haystack[:end], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
end = idx + len(needle) - 1
|
||||
}
|
||||
|
||||
//not found
|
||||
return -1
|
||||
}
|
||||
|
||||
@@ -160,15 +187,10 @@ func LastIndexOfAnyExcept1(in []rune, not rune) int {
|
||||
}
|
||||
|
||||
func LastIndexOfAny1(in []rune, find rune) int {
|
||||
for i := len(in) - 1; i >= 0; i-- {
|
||||
if in[i] == find {
|
||||
// found our char
|
||||
return i
|
||||
}
|
||||
if len(in) == 0 {
|
||||
return -1
|
||||
}
|
||||
|
||||
//not found
|
||||
return -1
|
||||
return lastIndexOfRuneBytes(runeSliceBytes(in), find)
|
||||
}
|
||||
|
||||
func LastIndexOfAnyInRange(in []rune, first, last rune) int {
|
||||
@@ -291,30 +313,66 @@ func foldASCII(c rune) rune {
|
||||
}
|
||||
|
||||
func IndexOf(in []rune, find []rune) int {
|
||||
/*
|
||||
Since we auto-gen the find code this shouldn't happen
|
||||
if len(find) == 0 {
|
||||
//special case
|
||||
return -1
|
||||
}*/
|
||||
if len(find) == 0 {
|
||||
return 0
|
||||
}
|
||||
if len(in) < len(find) {
|
||||
return -1
|
||||
}
|
||||
end := len(in) - len(find)
|
||||
first := find[0]
|
||||
//TODO: benchmark checking last char too or first two chars
|
||||
for i := 0; i <= end; i++ {
|
||||
// match start...check the rest
|
||||
if in[i] == first {
|
||||
// found our first char
|
||||
// check if the rest are equal
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
/*if slices.Equal(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}*/
|
||||
for i := 0; i <= end; {
|
||||
off := IndexOfAny1(in[i:end+1], first)
|
||||
if off < 0 {
|
||||
return -1
|
||||
}
|
||||
i += off
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
//not found
|
||||
func runeSliceBytes(in []rune) []byte {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
return unsafe.Slice((*byte)(unsafe.Pointer(&in[0])), len(in)*4)
|
||||
}
|
||||
|
||||
func indexOfRuneBytes(haystack []byte, find rune) int {
|
||||
needleRune := [1]rune{find}
|
||||
needle := runeSliceBytes(needleRune[:])
|
||||
start := 0
|
||||
for {
|
||||
idx := bytes.Index(haystack[start:], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
idx += start
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
start = idx + 1
|
||||
}
|
||||
}
|
||||
|
||||
func lastIndexOfRuneBytes(haystack []byte, find rune) int {
|
||||
needleRune := [1]rune{find}
|
||||
needle := runeSliceBytes(needleRune[:])
|
||||
end := len(haystack)
|
||||
for end >= 4 {
|
||||
idx := bytes.LastIndex(haystack[:end], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
end = idx + 3
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
|
||||
+47
-18
@@ -64,14 +64,30 @@ func (s AsciiSearchValues) IndexOfAnyExcept(chars []rune) int {
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s AsciiSearchValues) LastIndexOfAny(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
c := chars[i]
|
||||
if c > unicode.MaxASCII {
|
||||
continue
|
||||
}
|
||||
if s.set[c/64]&(1<<(c%64)) != 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s AsciiSearchValues) LastIndexOfAnyExcept(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
c := chars[i]
|
||||
if c > unicode.MaxASCII {
|
||||
return i
|
||||
}
|
||||
if s.set[c/64]&(1<<(c%64)) == 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type RuneSearchValues struct {
|
||||
@@ -79,37 +95,40 @@ type RuneSearchValues struct {
|
||||
}
|
||||
|
||||
func newRuneSearchValues(vals []rune) RuneSearchValues {
|
||||
//TODO: pre-calc the stuff we need to make each IndexOf go faster
|
||||
return RuneSearchValues{vals: vals}
|
||||
|
||||
}
|
||||
|
||||
func NewRuneSearchValues(vals string) RuneSearchValues {
|
||||
return newRuneSearchValues([]rune(vals))
|
||||
}
|
||||
|
||||
// return the first index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) IndexOfAny(chars []rune) int {
|
||||
//naive implementation
|
||||
//TODO: this
|
||||
return IndexOfAny(chars, s.vals)
|
||||
}
|
||||
|
||||
// return the first index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) IndexOfAnyExcept(chars []rune) int {
|
||||
//TODO: this
|
||||
return IndexOfAnyExcept(chars, s.vals)
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) LastIndexOfAny(chars []rune) int {
|
||||
panic("not implemented")
|
||||
if len(s.vals) == 1 {
|
||||
return LastIndexOfAny1(chars, s.vals[0])
|
||||
}
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
if slices.Contains(s.vals, chars[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) LastIndexOfAnyExcept(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
if !slices.Contains(s.vals, chars[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type StringSearchValues struct {
|
||||
@@ -145,11 +164,21 @@ func NewStringSearchValues(vals [][]rune, ignoreCase bool) StringSearchValues {
|
||||
}
|
||||
|
||||
func (s StringSearchValues) StartsWith(chars []rune) int {
|
||||
panic("not implemented")
|
||||
for _, val := range s.vals {
|
||||
if StartsWith(chars, val) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (s StringSearchValues) StartsWithIgnoreCase(chars []rune) int {
|
||||
panic("not implemented")
|
||||
for _, val := range s.vals {
|
||||
if StartsWithIgnoreCase(chars, val) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (s StringSearchValues) IndexOfAny(in []rune) int {
|
||||
|
||||
+62
-17
@@ -44,8 +44,9 @@ type Group struct {
|
||||
type Capture struct {
|
||||
// the original string
|
||||
text *matchText
|
||||
// RuneIndex is the position in the underlying rune slice where the first character of
|
||||
// captured substring was found. Even if you pass in a string this will be in Runes.
|
||||
// RuneIndex is the rune index in the original input where the capture starts.
|
||||
// For string input this counts runes from the start of that string, not of
|
||||
// any internally sliced decode buffer.
|
||||
RuneIndex int
|
||||
// RuneLength is the number of runes in the captured substring.
|
||||
RuneLength int
|
||||
@@ -55,24 +56,46 @@ type matchText struct {
|
||||
runes []rune
|
||||
input string
|
||||
hasStringInput bool
|
||||
runeOffset int // original-string rune index of runes[0]
|
||||
byteOffset int // original-string byte index of runes[0]
|
||||
byteOffsets []int
|
||||
byteOffsetsReady bool
|
||||
}
|
||||
|
||||
// String returns the captured text as a String
|
||||
// String returns the captured text. For string input it is a slice of the
|
||||
// original haystack: it does not allocate and keeps the original string alive.
|
||||
func (c *Capture) String() string {
|
||||
return string(c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength])
|
||||
if c.text == nil {
|
||||
return ""
|
||||
}
|
||||
if c.text.hasStringInput {
|
||||
start, length := c.ByteRange()
|
||||
return c.text.input[start : start+length]
|
||||
}
|
||||
start := c.runeSliceIndex()
|
||||
return string(c.text.runes[start : start+c.RuneLength])
|
||||
}
|
||||
|
||||
// Runes returns the captured text as a rune slice
|
||||
func (c *Capture) Runes() []rune {
|
||||
return c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength]
|
||||
if c.text == nil {
|
||||
return nil
|
||||
}
|
||||
start := c.runeSliceIndex()
|
||||
return c.text.runes[start : start+c.RuneLength]
|
||||
}
|
||||
|
||||
func (c *Capture) runeSliceIndex() int {
|
||||
if c.text == nil {
|
||||
return c.RuneIndex
|
||||
}
|
||||
return c.RuneIndex - c.text.runeOffset
|
||||
}
|
||||
|
||||
// ByteRange returns the UTF-8 byte index and byte length of the captured
|
||||
// substring. The first call lazily caches byte offsets on shared match text,
|
||||
// so it is not safe to call concurrently with ByteRange on another capture
|
||||
// from the same match until the cache has been initialized.
|
||||
// substring. Matches returned to callers have offsets computed when the match
|
||||
// is tidied, so concurrent ByteRange/String on captures of the same match is
|
||||
// then safe. Internal match objects may still initialize the cache on first use.
|
||||
func (c *Capture) ByteRange() (index, length int) {
|
||||
if c.text == nil {
|
||||
return c.RuneIndex, c.RuneLength
|
||||
@@ -85,24 +108,40 @@ func newMatchText(r []rune) *matchText {
|
||||
}
|
||||
|
||||
func newStringMatchText(input string, r []rune) *matchText {
|
||||
return &matchText{runes: r, input: input, hasStringInput: true}
|
||||
return newStringMatchTextAt(input, r, 0, 0)
|
||||
}
|
||||
|
||||
func newStringMatchTextAt(input string, r []rune, runeOffset, byteOffset int) *matchText {
|
||||
return &matchText{
|
||||
runes: r,
|
||||
input: input,
|
||||
hasStringInput: true,
|
||||
runeOffset: runeOffset,
|
||||
byteOffset: byteOffset,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *matchText) ensureByteOffsets() {
|
||||
if t == nil || t.byteOffsetsReady {
|
||||
return
|
||||
}
|
||||
t.byteOffsets = t.buildByteOffsets()
|
||||
t.byteOffsetsReady = true
|
||||
}
|
||||
|
||||
func (t *matchText) byteRange(runeIndex, runeLength int) (int, int) {
|
||||
if !t.byteOffsetsReady {
|
||||
t.byteOffsets = t.buildByteOffsets()
|
||||
t.byteOffsetsReady = true
|
||||
}
|
||||
localRuneIndex := runeIndex - t.runeOffset
|
||||
t.ensureByteOffsets()
|
||||
if t.byteOffsets == nil {
|
||||
return runeIndex, runeLength
|
||||
return t.byteOffset + localRuneIndex, runeLength
|
||||
}
|
||||
byteIndex := t.byteOffsets[runeIndex]
|
||||
return byteIndex, t.byteOffsets[runeIndex+runeLength] - byteIndex
|
||||
byteIndex := t.byteOffsets[localRuneIndex]
|
||||
return t.byteOffset + byteIndex, t.byteOffsets[localRuneIndex+runeLength] - byteIndex
|
||||
}
|
||||
|
||||
func (t *matchText) buildByteOffsets() []int {
|
||||
if t.hasStringInput {
|
||||
return stringByteOffsets(t.input)
|
||||
return stringByteOffsets(t.input[t.byteOffset:])
|
||||
}
|
||||
return runeByteOffsets(t.runes)
|
||||
}
|
||||
@@ -199,6 +238,9 @@ func (m *Match) tidy(textpos int) {
|
||||
m.capcount = m.matchcount[0]
|
||||
//copy our root capture to the list
|
||||
m.Captures = []Capture{m.Capture}
|
||||
if m.text != nil && m.text.hasStringInput {
|
||||
m.text.ensureByteOffsets()
|
||||
}
|
||||
|
||||
if m.balancing {
|
||||
// The idea here is that we want to compact all of our unbalanced captures. To do that we
|
||||
@@ -415,6 +457,9 @@ func newCapture(text *matchText, runeIndex, runeLength int) Capture {
|
||||
}
|
||||
|
||||
func setCaptureFields(c *Capture, runeIndex, runeLength int) {
|
||||
if c.text != nil {
|
||||
runeIndex += c.text.runeOffset
|
||||
}
|
||||
c.RuneIndex = runeIndex
|
||||
c.RuneLength = runeLength
|
||||
}
|
||||
|
||||
+14
-1
@@ -5,6 +5,7 @@ var (
|
||||
DefaultUnmarshalOptions = None
|
||||
// DefaultOptimizationOptions controls the default memory/performance trade-offs used by Compile.
|
||||
DefaultOptimizationOptions = OptimizationOptions{
|
||||
MaxBacktrackingStackSize: 100000,
|
||||
MaxCachedRuneBufferLength: 256 << 10,
|
||||
MaxCachedReplaceBufferLength: 256 << 10,
|
||||
MaxCachedReplacerDataEntries: 16,
|
||||
@@ -36,13 +37,17 @@ const (
|
||||
Unicode RegexOptions = 0x0400 // "u"
|
||||
)
|
||||
|
||||
// OptimizationOptions controls optional runtime caches and compile-time fast paths.
|
||||
// OptimizationOptions controls runtime limits, optional caches, and compile-time fast paths.
|
||||
//
|
||||
// For MaxBacktrackingStackSize, negative values allow unbounded growth.
|
||||
// For replacement data cache size fields, 0 disables persistent retention and
|
||||
// -1 means unbounded. For pooled buffer cache size fields, 0 disables pooling
|
||||
// and -1 allows all built-in size classes.
|
||||
// Defaults are intentionally bounded so Compile is safe for mixed-cardinality inputs.
|
||||
type OptimizationOptions struct {
|
||||
// MaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack.
|
||||
// Negative values disable the limit.
|
||||
MaxBacktrackingStackSize int
|
||||
// MaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool.
|
||||
MaxCachedRuneBufferLength int
|
||||
// MaxCachedReplaceBufferLength limits retained replacement output buffers in the shared size-classed pool.
|
||||
@@ -100,6 +105,14 @@ func newCompileConfig(options []CompileOption) compileConfig {
|
||||
return c
|
||||
}
|
||||
|
||||
// OptionMaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack.
|
||||
// Negative values disable the limit. A match that exceeds the limit returns ErrBacktrackingStackLimit.
|
||||
func OptionMaxBacktrackingStackSize(n int) CompileOption {
|
||||
return compileOptionFunc(func(c *compileConfig) {
|
||||
c.optimizations.MaxBacktrackingStackSize = n
|
||||
})
|
||||
}
|
||||
|
||||
// OptionMaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool.
|
||||
func OptionMaxCachedRuneBufferLength(n int) CompileOption {
|
||||
return compileOptionFunc(func(c *compileConfig) {
|
||||
|
||||
+64
-111
@@ -25,6 +25,8 @@ import (
|
||||
var (
|
||||
// DefaultMatchTimeout used when running regexp matches -- "forever"
|
||||
DefaultMatchTimeout = time.Duration(math.MaxInt64)
|
||||
// ErrBacktrackingStackLimit is returned when a match exceeds its configured backtracking stack size.
|
||||
ErrBacktrackingStackLimit = errors.New("regexp2: maximum backtracking stack size exceeded")
|
||||
)
|
||||
|
||||
// Regexp is the representation of a compiled regular expression.
|
||||
@@ -59,7 +61,12 @@ type Regexp struct {
|
||||
// hook points to override runner functions
|
||||
findFirstChar func(r *Runner) bool
|
||||
execute func(r *Runner) error
|
||||
executeQuick func(r *Runner) error
|
||||
stringPrefixFilter StringPrefixFilter
|
||||
quickCode *syntax.Code // bool-only program with unobservable captures removed
|
||||
// leftContextRunes is used when code is nil (registered engines).
|
||||
// The interpreter reads the same value from code.LeftContextRunes.
|
||||
leftContextRunes int
|
||||
}
|
||||
|
||||
// Compile parses a regular expression and returns, if successful,
|
||||
@@ -107,6 +114,7 @@ func compile(expr string, c compileConfig) (*Regexp, error) {
|
||||
capslist: tree.Caplist,
|
||||
capsize: code.Capsize,
|
||||
code: code,
|
||||
quickCode: makeQuickCode(code),
|
||||
MatchTimeout: DefaultMatchTimeout,
|
||||
optimizations: c.optimizations,
|
||||
}
|
||||
@@ -115,6 +123,18 @@ func compile(expr string, c compileConfig) (*Regexp, error) {
|
||||
return re, nil
|
||||
}
|
||||
|
||||
func makeQuickCode(code *syntax.Code) *syntax.Code {
|
||||
if code == nil || len(code.QuickCodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
quick := *code
|
||||
quick.Codes = code.QuickCodes
|
||||
quick.Dispatches = code.QuickDispatches
|
||||
quick.QuickCodes = nil
|
||||
quick.QuickDispatches = nil
|
||||
return &quick
|
||||
}
|
||||
|
||||
// MustCompile is like Compile but panics if the expression cannot be parsed.
|
||||
// It simplifies safe initialization of global variables holding compiled regular
|
||||
// expressions.
|
||||
@@ -225,17 +245,12 @@ func (re *Regexp) FindStringMatch(s string) (*Match, error) {
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r, runeStart := re.getRunesAndStart(s, startAt)
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
return re.run(false, runeStart, r, newStringMatchText(s, r))
|
||||
return re.findDecodedStringMatch(s, startAt)
|
||||
}
|
||||
|
||||
// FindRunesMatch searches the input rune slice for a Regexp match
|
||||
func (re *Regexp) FindRunesMatch(r []rune) (*Match, error) {
|
||||
return re.run(false, -1, r, newMatchText(r))
|
||||
return re.run(false, -1, -1, r, newMatchText(r))
|
||||
}
|
||||
|
||||
// FindStringMatchStartingAt searches the input string for a Regexp match starting at the startAt index
|
||||
@@ -247,19 +262,22 @@ func (re *Regexp) FindStringMatchStartingAt(s string, startAt int) (*Match, erro
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return re.findDecodedStringMatch(s, startAt)
|
||||
}
|
||||
|
||||
r, startAt := re.getRunesAndStart(s, startAt)
|
||||
if startAt == -1 {
|
||||
// we didn't find our start index in the string -- that's a problem
|
||||
return nil, errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
|
||||
return re.run(false, startAt, r, newStringMatchText(s, r))
|
||||
func (re *Regexp) findDecodedStringMatch(s string, startAt int) (*Match, error) {
|
||||
// Returned matches retain their rune data, so this path must not consume a
|
||||
// pooled buffer that can never be returned.
|
||||
d := re.decodeStringInput(s, startAt, false)
|
||||
runner := re.getRunner()
|
||||
defer re.putRunner(runner)
|
||||
text := newStringMatchTextAt(s, d.runes, d.runeOffset, d.byteOffset)
|
||||
return runner.scan(d.runes, text, d.runeStart, -1, false, re.MatchTimeout)
|
||||
}
|
||||
|
||||
// FindRunesMatchStartingAt searches the input rune slice for a Regexp match starting at the startAt index
|
||||
func (re *Regexp) FindRunesMatchStartingAt(r []rune, startAt int) (*Match, error) {
|
||||
return re.run(false, startAt, r, newMatchText(r))
|
||||
return re.run(false, startAt, -1, r, newMatchText(r))
|
||||
}
|
||||
|
||||
// FindAllStringIndex returns a slice of byte index pairs identifying all
|
||||
@@ -277,33 +295,22 @@ func (re *Regexp) FindAllStringIndex(s string, n int) ([][]int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
d := re.decodeStringInput(s, startAt, true)
|
||||
runner := re.getRunner()
|
||||
var input []rune
|
||||
var pooledInput *[]rune
|
||||
runeStart := 0
|
||||
if startAt == 0 {
|
||||
input, pooledInput = runner.decodeString(s)
|
||||
} else {
|
||||
input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt)
|
||||
}
|
||||
defer func() {
|
||||
re.putRunner(runner)
|
||||
if pooledInput != nil {
|
||||
*pooledInput = input
|
||||
pooledRuneBuffers.put(pooledInput)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
|
||||
byteOffsets := newStringByteMapper(s)
|
||||
return re.findAllRunesIndex(runner, input, runeStart, n, func(runeIndex, runeLength int) (int, int) {
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
return re.findAllRunesIndex(runner, d.runes, d.runeStart, n, func(runeIndex, runeLength int) (int, int) {
|
||||
if byteOffsets == nil {
|
||||
return runeIndex, runeIndex + runeLength
|
||||
return d.byteOffset + runeIndex, d.byteOffset + runeIndex + runeLength
|
||||
}
|
||||
return byteOffsets.byteIndex(runeIndex), byteOffsets.byteIndex(runeIndex + runeLength)
|
||||
start := runeIndex + d.runeOffset
|
||||
return byteOffsets.byteIndex(start), byteOffsets.byteIndex(start + runeLength)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -321,6 +328,9 @@ func (re *Regexp) FindAllRunesIndex(r []rune, n int) ([][]int, error) {
|
||||
if re.RightToLeft() {
|
||||
startAt = len(r)
|
||||
}
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
return re.findAllRunesIndex(runner, r, startAt, n, func(runeIndex, runeLength int) (int, int) {
|
||||
return runeIndex, runeIndex + runeLength
|
||||
})
|
||||
@@ -335,8 +345,9 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int
|
||||
}
|
||||
|
||||
prevEnd := -1
|
||||
previousMatchLength := -1
|
||||
for n != 0 {
|
||||
m, err := runner.scan(input, nil, startAt, true, re.MatchTimeout)
|
||||
m, err := runner.scan(input, nil, startAt, previousMatchLength, true, re.MatchTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -344,34 +355,19 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int
|
||||
break
|
||||
}
|
||||
|
||||
if m.RuneLength != 0 || m.RuneIndex != prevEnd {
|
||||
start, end := makeIndex(m.RuneIndex, m.RuneLength)
|
||||
localIndex := m.runeSliceIndex()
|
||||
if m.RuneLength != 0 || localIndex != prevEnd {
|
||||
start, end := makeIndex(localIndex, m.RuneLength)
|
||||
flat = append(flat, start, end)
|
||||
out = append(out, flat[len(flat)-2:len(flat):len(flat)])
|
||||
prevEnd = m.RuneIndex + m.RuneLength
|
||||
prevEnd = localIndex + m.RuneLength
|
||||
if n > 0 {
|
||||
n--
|
||||
}
|
||||
}
|
||||
|
||||
startAt = m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if re.RightToLeft() {
|
||||
if m.textpos == 0 {
|
||||
break
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt--
|
||||
}
|
||||
} else {
|
||||
if m.textpos == len(input) {
|
||||
break
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt++
|
||||
}
|
||||
}
|
||||
}
|
||||
previousMatchLength = m.RuneLength
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -420,29 +416,7 @@ func (re *Regexp) FindNextMatch(m *Match) (*Match, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// If previous match was empty, advance by one before matching to prevent
|
||||
// infinite loop
|
||||
startAt := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if re.RightToLeft() {
|
||||
if m.textpos == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt--
|
||||
}
|
||||
} else {
|
||||
if m.textpos == len(m.text.runes) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if startAt == m.textstart {
|
||||
startAt++
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return re.run(false, startAt, m.text.runes, m.text)
|
||||
return re.run(false, m.textpos, m.RuneLength, m.text.runes, m.text)
|
||||
}
|
||||
|
||||
// MatchString return true if the string matches the regex
|
||||
@@ -469,12 +443,16 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) {
|
||||
var pooledInput *[]rune
|
||||
runeStart := 0
|
||||
if startAt <= 0 {
|
||||
input, pooledInput = runner.decodeString(s)
|
||||
// Common path: decode the whole string without start/offset work.
|
||||
input, pooledInput = decodeString(s, re.optimizations.MaxCachedRuneBufferLength)
|
||||
if re.RightToLeft() {
|
||||
runeStart = len(input)
|
||||
}
|
||||
} else {
|
||||
input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt)
|
||||
d := decodeInput(s, startAt, re.decodeFrom(s, startAt), re.optimizations.MaxCachedRuneBufferLength, false)
|
||||
input = d.runes
|
||||
pooledInput = d.pooled
|
||||
runeStart = d.runeStart
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
@@ -486,46 +464,21 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) {
|
||||
pooledRuneBuffers.put(pooledInput)
|
||||
}
|
||||
}()
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
|
||||
m, err := runner.scan(input, nil, runeStart, true, re.MatchTimeout)
|
||||
m, err := runner.scan(input, nil, runeStart, -1, true, re.MatchTimeout)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return m != nil, nil
|
||||
}
|
||||
|
||||
func (re *Regexp) getRunesAndStart(s string, startAt int) ([]rune, int) {
|
||||
if startAt < 0 {
|
||||
if re.RightToLeft() {
|
||||
r := getRunes(s)
|
||||
return r, len(r)
|
||||
}
|
||||
return getRunes(s), 0
|
||||
}
|
||||
ret := make([]rune, len(s))
|
||||
i := 0
|
||||
runeIdx := -1
|
||||
for strIdx, r := range s {
|
||||
if strIdx == startAt {
|
||||
runeIdx = i
|
||||
}
|
||||
ret[i] = r
|
||||
i++
|
||||
}
|
||||
if startAt == len(s) {
|
||||
runeIdx = i
|
||||
}
|
||||
return ret[:i], runeIdx
|
||||
}
|
||||
|
||||
func getRunes(s string) []rune {
|
||||
return []rune(s)
|
||||
}
|
||||
|
||||
// MatchRunes return true if the runes matches the regex
|
||||
// error will be set if a timeout occurs
|
||||
func (re *Regexp) MatchRunes(r []rune) (bool, error) {
|
||||
m, err := re.run(true, -1, r, nil)
|
||||
m, err := re.run(true, -1, -1, r, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
+24
-7
@@ -11,13 +11,23 @@ type RuntimeEngineData struct {
|
||||
CapSize int // size of the capture array
|
||||
FindFirstChar func(*Runner) bool // generated candidate search
|
||||
Execute func(*Runner) error
|
||||
StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input
|
||||
ExecuteQuick func(*Runner) error // optional bool-only execution with unobservable captures removed
|
||||
StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input
|
||||
// LeftContextKnown reports that LeftContextRunes was computed by the
|
||||
// code generator. If it is false, decoded string input is never sliced;
|
||||
// older generated engines omit the field and must keep the full string.
|
||||
LeftContextKnown bool
|
||||
// LeftContextRunes is how many runes before a candidate start matching
|
||||
// may inspect. 0 means none, 1 means a single previous rune, and -1
|
||||
// means do not slice (lookbehind or \G). Ignored unless LeftContextKnown.
|
||||
LeftContextRunes int
|
||||
}
|
||||
|
||||
type cacheKey struct {
|
||||
pattern string
|
||||
opt RegexOptions
|
||||
maintainCaptureOrder bool
|
||||
pattern string
|
||||
opt RegexOptions
|
||||
maintainCaptureOrder bool
|
||||
maxBacktrackingStackSize int
|
||||
}
|
||||
|
||||
func RegisterEngine(pattern string, engine RuntimeEngineData, options ...CompileOption) {
|
||||
@@ -28,6 +38,10 @@ func RegisterEngine(pattern string, engine RuntimeEngineData, options ...Compile
|
||||
}
|
||||
|
||||
func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData) *Regexp {
|
||||
leftContext := -1
|
||||
if engine.LeftContextKnown {
|
||||
leftContext = engine.LeftContextRunes
|
||||
}
|
||||
re := &Regexp{
|
||||
pattern: pattern,
|
||||
options: c.regexOptions,
|
||||
@@ -40,7 +54,9 @@ func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData)
|
||||
optimizations: c.optimizations,
|
||||
findFirstChar: engine.FindFirstChar,
|
||||
execute: engine.Execute,
|
||||
executeQuick: engine.ExecuteQuick,
|
||||
stringPrefixFilter: engine.StringPrefixFilter,
|
||||
leftContextRunes: leftContext,
|
||||
}
|
||||
re.initCaches()
|
||||
return re
|
||||
@@ -58,9 +74,10 @@ func getEngineRegexp(pattern string, c compileConfig) *Regexp {
|
||||
|
||||
func cacheKeyFromConfig(pattern string, c compileConfig) cacheKey {
|
||||
return cacheKey{
|
||||
pattern: pattern,
|
||||
opt: c.regexOptions,
|
||||
maintainCaptureOrder: c.maintainCaptureOrder,
|
||||
pattern: pattern,
|
||||
opt: c.regexOptions,
|
||||
maintainCaptureOrder: c.maintainCaptureOrder,
|
||||
maxBacktrackingStackSize: c.optimizations.MaxBacktrackingStackSize,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-49
@@ -87,15 +87,15 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
text := m.text.runes
|
||||
|
||||
if !regex.RightToLeft() {
|
||||
prevat := 0
|
||||
for m != nil {
|
||||
if m.RuneIndex != prevat {
|
||||
buf.WriteString(string(text[prevat:m.RuneIndex]))
|
||||
start, end := matchInputSpan(m)
|
||||
if start > prevat {
|
||||
buf.WriteString(input[prevat:start])
|
||||
}
|
||||
prevat = m.RuneIndex + m.RuneLength
|
||||
prevat = end
|
||||
buf.WriteString(evaluator(*m))
|
||||
|
||||
count--
|
||||
@@ -108,18 +108,19 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
}
|
||||
|
||||
if prevat < len(text) {
|
||||
buf.WriteString(string(text[prevat:]))
|
||||
if prevat < len(input) {
|
||||
buf.WriteString(input[prevat:])
|
||||
}
|
||||
} else {
|
||||
prevat := len(text)
|
||||
prevat := len(input)
|
||||
var al []string
|
||||
|
||||
for m != nil {
|
||||
if m.RuneIndex+m.RuneLength != prevat {
|
||||
al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat]))
|
||||
start, end := matchInputSpan(m)
|
||||
if end < prevat {
|
||||
al = append(al, input[end:prevat])
|
||||
}
|
||||
prevat = m.RuneIndex
|
||||
prevat = start
|
||||
al = append(al, evaluator(*m))
|
||||
|
||||
count--
|
||||
@@ -133,7 +134,7 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
|
||||
if prevat > 0 {
|
||||
buf.WriteString(string(text[:prevat]))
|
||||
buf.WriteString(input[:prevat])
|
||||
}
|
||||
|
||||
for i := len(al) - 1; i >= 0; i-- {
|
||||
@@ -144,20 +145,29 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// matchInputSpan returns the UTF-8 byte range of m in its original string input.
|
||||
func matchInputSpan(m *Match) (start, end int) {
|
||||
if m.text != nil && m.text.hasStringInput {
|
||||
start, length := m.ByteRange()
|
||||
return start, start + length
|
||||
}
|
||||
return m.RuneIndex, m.RuneIndex + m.RuneLength
|
||||
}
|
||||
|
||||
func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, startAt, count int) (string, error) {
|
||||
if startAt > len(input) {
|
||||
return "", errors.New("startAt must be less than the length of the input string")
|
||||
}
|
||||
|
||||
runner := regex.getRunner()
|
||||
text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt)
|
||||
d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false)
|
||||
text := d.runes
|
||||
textInfo := newStringMatchText(input, text)
|
||||
defer func() {
|
||||
regex.putRunner(runner)
|
||||
if pooledText != nil {
|
||||
pooledRuneBuffers.put(pooledText)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
runeStart := d.runeStart
|
||||
if startAt >= 0 && runeStart < 0 {
|
||||
return "", errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
@@ -165,7 +175,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
runeStart = 0
|
||||
}
|
||||
|
||||
m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout)
|
||||
m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -184,10 +194,11 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
compactBalancedMatches(m)
|
||||
}
|
||||
|
||||
if m.RuneIndex != prevat {
|
||||
writeRunes(buf, text, prevat, m.RuneIndex)
|
||||
local := m.runeSliceIndex()
|
||||
if local != prevat {
|
||||
writeRunes(buf, text, prevat, local)
|
||||
}
|
||||
prevat = m.RuneIndex + m.RuneLength
|
||||
prevat = local + m.RuneLength
|
||||
replacementImpl(data, buf, m)
|
||||
|
||||
count--
|
||||
@@ -195,15 +206,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
break
|
||||
}
|
||||
|
||||
scanStart := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if scanStart >= len(text) {
|
||||
break
|
||||
}
|
||||
scanStart++
|
||||
}
|
||||
|
||||
m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout)
|
||||
m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -221,14 +224,14 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
}
|
||||
|
||||
runner := regex.getRunner()
|
||||
text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt)
|
||||
d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false)
|
||||
text := d.runes
|
||||
textInfo := newStringMatchText(input, text)
|
||||
defer func() {
|
||||
regex.putRunner(runner)
|
||||
if pooledText != nil {
|
||||
pooledRuneBuffers.put(pooledText)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
runeStart := d.runeStart
|
||||
if startAt >= 0 && runeStart < 0 {
|
||||
return "", errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
@@ -236,7 +239,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
runeStart = len(text)
|
||||
}
|
||||
|
||||
m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout)
|
||||
m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -257,10 +260,11 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
compactBalancedMatches(m)
|
||||
}
|
||||
|
||||
if m.RuneIndex+m.RuneLength != prevat {
|
||||
al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat]))
|
||||
local := m.runeSliceIndex()
|
||||
if local+m.RuneLength != prevat {
|
||||
al = append(al, string(text[local+m.RuneLength:prevat]))
|
||||
}
|
||||
prevat = m.RuneIndex
|
||||
prevat = local
|
||||
replacementImplRTL(data, &al, m)
|
||||
|
||||
count--
|
||||
@@ -268,15 +272,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
break
|
||||
}
|
||||
|
||||
scanStart := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if scanStart <= 0 {
|
||||
break
|
||||
}
|
||||
scanStart--
|
||||
}
|
||||
|
||||
m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout)
|
||||
m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -303,11 +299,12 @@ func replacementImpl(data *syntax.ReplacerData, buf *bytes.Buffer, m *Match) {
|
||||
} else {
|
||||
switch -replaceSpecials - 1 - r { // special insertion patterns
|
||||
case replaceLeftPortion:
|
||||
for i := 0; i < m.RuneIndex; i++ {
|
||||
end := m.runeSliceIndex()
|
||||
for i := 0; i < end; i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceRightPortion:
|
||||
for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceLastGroup:
|
||||
@@ -335,11 +332,12 @@ func replacementImplRTL(data *syntax.ReplacerData, al *[]string, m *Match) {
|
||||
} else {
|
||||
switch -replaceSpecials - 1 - r { // special insertion patterns
|
||||
case replaceLeftPortion:
|
||||
for i := 0; i < m.RuneIndex; i++ {
|
||||
end := m.runeSliceIndex()
|
||||
for i := 0; i < end; i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceRightPortion:
|
||||
for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceLastGroup:
|
||||
|
||||
+239
-75
@@ -74,8 +74,10 @@ type Runner struct {
|
||||
//
|
||||
// quick is usually false, but can be true to not return matches, just put it in caches.
|
||||
// textstart is -1 to start at the "beginning" (depending on Right-To-Left), otherwise an index in input.
|
||||
// previousMatchLength is -1 for an initial scan. A zero value advances the current scan position while
|
||||
// preserving textstart for anchors such as \G.
|
||||
// textInfo is nil for quick scans that do not need returned capture text metadata.
|
||||
func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchText) (*Match, error) {
|
||||
func (re *Regexp) run(quick bool, textstart, previousMatchLength int, input []rune, textInfo *matchText) (*Match, error) {
|
||||
|
||||
// get a cached runner
|
||||
runner := re.getRunner()
|
||||
@@ -88,8 +90,11 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe
|
||||
textstart = 0
|
||||
}
|
||||
}
|
||||
if quick && textInfo == nil && re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
|
||||
return runner.scan(input, textInfo, textstart, quick, re.MatchTimeout)
|
||||
return runner.scan(input, textInfo, textstart, previousMatchLength, quick, re.MatchTimeout)
|
||||
}
|
||||
|
||||
// Scans the string to find the first match. Uses the Match object
|
||||
@@ -108,13 +113,16 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe
|
||||
// used as a boolean result and capture text is intentionally unavailable. If
|
||||
// we collapsed down to just textInfo it would "escape" and hit the GC for fast
|
||||
// scans without captures.
|
||||
func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, timeout time.Duration) (*Match, error) {
|
||||
func (r *Runner) scan(rt []rune, textInfo *matchText, textstart, previousMatchLength int, quick bool, timeout time.Duration) (*Match, error) {
|
||||
r.timeout = timeout
|
||||
r.ignoreTimeout = (time.Duration(math.MaxInt64) == timeout)
|
||||
r.debug = r.re.Debug()
|
||||
r.Runtextstart = textstart
|
||||
r.Runtext = rt
|
||||
r.Runtextend = len(rt)
|
||||
// Some internal callers use quick match tidying while still consuming
|
||||
// capture data (notably replacement). Capture elision is only safe when no
|
||||
// match text metadata was requested.
|
||||
|
||||
stoppos := r.Runtextend
|
||||
bump := 1
|
||||
@@ -130,6 +138,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
// setup our scanner functions
|
||||
findFirstChar := r.re.findFirstChar
|
||||
execute := r.re.execute
|
||||
if quick && textInfo == nil && r.re.executeQuick != nil {
|
||||
execute = r.re.executeQuick
|
||||
}
|
||||
if findFirstChar == nil {
|
||||
findFirstChar = findFirstCharDefault
|
||||
}
|
||||
@@ -144,6 +155,16 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
|
||||
r.initMatch(textInfo)
|
||||
|
||||
// An empty previous match must not be returned again. Keep Runtextstart at
|
||||
// the previous match position for \G, but move the candidate scan position.
|
||||
if previousMatchLength == 0 {
|
||||
if r.Runtextpos == stoppos {
|
||||
r.tidyMatch(true)
|
||||
return nil, nil
|
||||
}
|
||||
r.Runtextpos += bump
|
||||
}
|
||||
|
||||
r.startTimeoutWatch()
|
||||
for {
|
||||
if minRequiredLength > 0 {
|
||||
@@ -208,8 +229,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
|
||||
func executeDefault(r *Runner) error {
|
||||
|
||||
r.goTo(0)
|
||||
|
||||
if err := r.goTo(0); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
|
||||
if r.debug {
|
||||
@@ -230,7 +252,59 @@ func executeDefault(r *Runner) error {
|
||||
//noop
|
||||
|
||||
case syntax.Goto:
|
||||
r.goTo(r.operand(0))
|
||||
if err := r.goTo(r.operand(0)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Dispatch:
|
||||
// Dispatch only peeks at the next rune. It consumes it after finding a
|
||||
// matching branch, so a failed dispatch leaves the input position alone.
|
||||
if r.forwardchars() < 1 {
|
||||
break
|
||||
}
|
||||
// Pick the next rune in the current execution direction.
|
||||
pos := r.Runtextpos
|
||||
if r.rightToLeft {
|
||||
pos--
|
||||
}
|
||||
ch := r.Runtext[pos]
|
||||
tableIndex := r.operand(0)
|
||||
table := &r.code.Dispatches[tableIndex]
|
||||
branch := -1
|
||||
if ch >= 0 && ch < 128 {
|
||||
if table.ASCII != nil {
|
||||
// Larger dispatches use a direct ASCII branch lookup.
|
||||
branch = int(table.ASCII[ch]) - 1
|
||||
} else {
|
||||
// Smaller dispatches use two compact ASCII bitmasks per set.
|
||||
word := int(ch >> 6)
|
||||
bit := uint64(1) << (ch & 63)
|
||||
for i := range table.Sets {
|
||||
if table.ASCIIMasks[i*2+word]&bit != 0 {
|
||||
branch = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-ASCII runes fall back to the complete character sets.
|
||||
for i, setIndex := range table.Sets {
|
||||
if r.code.Sets[setIndex].CharIn(ch) {
|
||||
branch = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if branch < 0 {
|
||||
break
|
||||
}
|
||||
// The selected branch starts with this rune, so consume it and jump
|
||||
// directly to the rest of that branch.
|
||||
r.Runtextpos += r.bump()
|
||||
if err := r.goTo(table.Branches[branch]); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Testref:
|
||||
@@ -248,7 +322,9 @@ func executeDefault(r *Runner) error {
|
||||
case syntax.Lazybranch | syntax.Back:
|
||||
r.trackPop()
|
||||
r.textto(r.trackPeek())
|
||||
r.goTo(r.operand(0))
|
||||
if err := r.goTo(r.operand(0)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Setmark:
|
||||
@@ -307,9 +383,11 @@ func executeDefault(r *Runner) error {
|
||||
matched := r.textPos() - r.stackPeek()
|
||||
|
||||
if matched != 0 { // Nonempty match -> loop now
|
||||
r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos
|
||||
r.stackPush(r.textPos()) // Make new mark
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos
|
||||
r.stackPush(r.textPos()) // Make new mark
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
} else { // Empty match -> straight now
|
||||
r.trackPushNeg1(r.stackPeek()) // Save old mark
|
||||
r.advance(1) // Straight
|
||||
@@ -364,10 +442,12 @@ func executeDefault(r *Runner) error {
|
||||
|
||||
r.trackPopN(2)
|
||||
pos := r.trackPeekN(1)
|
||||
r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark
|
||||
r.stackPush(pos) // Make new mark
|
||||
r.textto(pos) // Recall position
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark
|
||||
r.stackPush(pos) // Make new mark
|
||||
r.textto(pos) // Recall position
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Lazybranchmark | syntax.Back2:
|
||||
@@ -413,9 +493,11 @@ func executeDefault(r *Runner) error {
|
||||
r.trackPushNeg2(mark, count) // Save old mark, count
|
||||
r.advance(2) // Straight
|
||||
} else { // Nonempty match -> count+loop now
|
||||
r.trackPush1(mark) // remember mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPush1(mark) // remember mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
|
||||
@@ -452,9 +534,11 @@ func executeDefault(r *Runner) error {
|
||||
count := r.stackPeekN(1)
|
||||
|
||||
if count < 0 { // Negative count -> loop now
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
} else { // Nonneg count -> straight now
|
||||
r.trackPush3(mark, count, r.textPos()) // Save mark, count, position
|
||||
r.advance(2) // Straight
|
||||
@@ -472,10 +556,12 @@ func executeDefault(r *Runner) error {
|
||||
textpos := r.trackPeekN(2)
|
||||
|
||||
if r.trackPeekN(1) < r.operand(1) && textpos != mark { // Under limit and not empty match -> loop
|
||||
r.textto(textpos) // Recall position
|
||||
r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.textto(textpos) // Recall position
|
||||
r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
continue
|
||||
} else { // Max loops or empty match -> backtrack
|
||||
r.stackPush2(r.trackPeek(), r.trackPeekN(1)) // Recall old mark, count
|
||||
@@ -648,6 +734,13 @@ func executeDefault(r *Runner) error {
|
||||
r.advance(1)
|
||||
continue
|
||||
|
||||
case syntax.Grapheme:
|
||||
if !r.TryMatchGrapheme(r.rightToLeft) {
|
||||
break
|
||||
}
|
||||
r.advance(0)
|
||||
continue
|
||||
|
||||
case syntax.Ref:
|
||||
|
||||
capnum := r.operand(0)
|
||||
@@ -935,18 +1028,21 @@ func executeDefault(r *Runner) error {
|
||||
;
|
||||
|
||||
// "break Backward" comes here:
|
||||
r.backtrack()
|
||||
if err := r.backtrack(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// increase the size of stack and track storage
|
||||
func (r *Runner) ensureStorage() {
|
||||
func (r *Runner) ensureStorage() error {
|
||||
if r.Runstackpos < r.runtrackcount*4 {
|
||||
doubleIntSlice(&r.runstack, &r.Runstackpos)
|
||||
}
|
||||
if r.Runtrackpos < r.runtrackcount*4 {
|
||||
doubleIntSlice(&r.runtrack, &r.Runtrackpos)
|
||||
if r.Runtrackpos < r.runtrackcount*4 && !r.growTrack() {
|
||||
return ErrBacktrackingStackLimit
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) ensureStack(plus int) {
|
||||
@@ -990,14 +1086,17 @@ func (r *Runner) advance(i int) {
|
||||
r.setOperator(r.code.Codes[r.codepos])
|
||||
}
|
||||
|
||||
func (r *Runner) goTo(newpos int) {
|
||||
func (r *Runner) goTo(newpos int) error {
|
||||
// when branching backward or in place, ensure storage
|
||||
if newpos <= r.codepos {
|
||||
r.ensureStorage()
|
||||
if err := r.ensureStorage(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
r.setOperator(r.code.Codes[newpos])
|
||||
r.codepos = newpos
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) textto(newpos int) {
|
||||
@@ -1016,11 +1115,47 @@ func (r *Runner) textPos() int {
|
||||
return r.Runtextpos
|
||||
}
|
||||
|
||||
// TryMatchGrapheme consumes one Unicode extended grapheme cluster in the
|
||||
// runner's current direction. It is exported for regexp2cg-generated engines.
|
||||
func (r *Runner) TryMatchGrapheme(rightToLeft bool) bool {
|
||||
var boundary int
|
||||
if rightToLeft {
|
||||
boundary = syntax.PreviousGraphemeClusterBoundary(r.Runtext, r.Runtextpos)
|
||||
} else {
|
||||
boundary = syntax.NextGraphemeClusterBoundary(r.Runtext, r.Runtextpos)
|
||||
}
|
||||
if boundary < 0 {
|
||||
return false
|
||||
}
|
||||
r.Runtextpos = boundary
|
||||
return true
|
||||
}
|
||||
|
||||
// push onto the backtracking stack
|
||||
func (r *Runner) trackpos() int {
|
||||
return len(r.runtrack) - r.Runtrackpos
|
||||
}
|
||||
|
||||
func (r *Runner) growTrack() bool {
|
||||
oldLen := len(r.runtrack)
|
||||
newLen := oldLen * 2
|
||||
if newLen == 0 {
|
||||
newLen = 1
|
||||
}
|
||||
if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && newLen > limit {
|
||||
newLen = limit
|
||||
}
|
||||
if newLen <= oldLen {
|
||||
return false
|
||||
}
|
||||
|
||||
newTrack := make([]int, newLen)
|
||||
copy(newTrack[newLen-oldLen:], r.runtrack)
|
||||
r.Runtrackpos += newLen - oldLen
|
||||
r.runtrack = newTrack
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *Runner) trackPush() {
|
||||
r.Runtrackpos--
|
||||
r.runtrack[r.Runtrackpos] = r.codepos
|
||||
@@ -1069,7 +1204,7 @@ func (r *Runner) trackPushNeg2(I1, I2 int) {
|
||||
r.runtrack[r.Runtrackpos] = -r.codepos
|
||||
}
|
||||
|
||||
func (r *Runner) backtrack() {
|
||||
func (r *Runner) backtrack() error {
|
||||
newpos := r.runtrack[r.Runtrackpos]
|
||||
r.Runtrackpos++
|
||||
|
||||
@@ -1090,10 +1225,13 @@ func (r *Runner) backtrack() {
|
||||
|
||||
// When branching backward, ensure storage
|
||||
if newpos < r.codepos {
|
||||
r.ensureStorage()
|
||||
if err := r.ensureStorage(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
r.codepos = newpos
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) setOperator(op int) {
|
||||
@@ -1397,7 +1535,8 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
switch r.code.FindOptimizations.FindMode {
|
||||
opts := r.code.FindOptimizations
|
||||
switch opts.FindMode {
|
||||
case syntax.TrailingAnchor_FixedLength_LeftToRight_End,
|
||||
syntax.LeadingString_OrdinalIgnoreCase_LeftToRight,
|
||||
syntax.LeadingStrings_LeftToRight,
|
||||
@@ -1408,6 +1547,13 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool {
|
||||
syntax.LiteralAfterLoop_LeftToRight,
|
||||
syntax.RequiredLandmarkChain_LeftToRight:
|
||||
return true
|
||||
case syntax.LeadingSet_LeftToRight:
|
||||
// General Unicode sets already have a direct fallback loop below.
|
||||
// Large enumerated sets are also faster through the set's ASCII bitmap
|
||||
// than through the linear IndexOfAny helper.
|
||||
return len(opts.FixedDistanceSets) > 0 &&
|
||||
((len(opts.FixedDistanceSets[0].Chars) > 0 && len(opts.FixedDistanceSets[0].Chars) <= 5) ||
|
||||
opts.FixedDistanceSets[0].Range != nil)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -1429,10 +1575,10 @@ func findFirstCharOptimized(r *Runner) (handled bool, found bool) {
|
||||
case syntax.LeadingString_OrdinalIgnoreCase_LeftToRight:
|
||||
return true, findLeadingStringLeftToRight(r, []rune(opts.LeadingPrefix), true)
|
||||
case syntax.LeadingStrings_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, false)
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, false)
|
||||
case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, true)
|
||||
case syntax.FixedDistanceSets_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, true)
|
||||
case syntax.LeadingSet_LeftToRight, syntax.FixedDistanceSets_LeftToRight:
|
||||
return true, findFixedDistanceSetsLeftToRight(r, opts.FixedDistanceSets)
|
||||
case syntax.FixedDistanceChar_LeftToRight:
|
||||
return true, findFixedDistanceCharLeftToRight(r, opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance)
|
||||
@@ -1487,29 +1633,69 @@ func findLeadingStringLeftToRight(r *Runner, prefix []rune, ignoreCase bool) boo
|
||||
return true
|
||||
}
|
||||
|
||||
func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, ignoreCase bool) bool {
|
||||
func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, firstRunes []rune, ignoreCase bool) bool {
|
||||
if len(prefixes) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for start := r.Runtextpos; start <= latestPossibleStart(r); start++ {
|
||||
for _, prefix := range prefixes {
|
||||
if ignoreCase {
|
||||
if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) {
|
||||
// Unicode ordinal-ignore-case matching has more possible first-rune folds
|
||||
// than a small precomputed set can safely represent. Keep its conservative
|
||||
// position-by-position scan; the common case-sensitive path skips directly
|
||||
// between possible first runes.
|
||||
if ignoreCase || len(firstRunes) == 0 {
|
||||
for start := r.Runtextpos; start <= latestPossibleStart(r); start++ {
|
||||
for _, prefix := range prefixes {
|
||||
if ignoreCase {
|
||||
if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
} else if helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
} else if helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
}
|
||||
}
|
||||
r.Runtextpos = r.Runtextend
|
||||
return false
|
||||
}
|
||||
|
||||
latest := min(latestPossibleStart(r), r.Runtextend-1)
|
||||
for searchAt := r.Runtextpos; searchAt <= latest; {
|
||||
offset := indexOfAnyRunes(r.Runtext[searchAt:latest+1], firstRunes)
|
||||
if offset < 0 {
|
||||
break
|
||||
}
|
||||
start := searchAt + offset
|
||||
first := r.Runtext[start]
|
||||
for _, prefix := range prefixes {
|
||||
if len(prefix) > 0 && prefix[0] == first && helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
}
|
||||
searchAt = start + 1
|
||||
}
|
||||
|
||||
r.Runtextpos = r.Runtextend
|
||||
return false
|
||||
}
|
||||
|
||||
func indexOfAnyRunes(input, find []rune) int {
|
||||
switch len(find) {
|
||||
case 0:
|
||||
return -1
|
||||
case 1:
|
||||
return helpers.IndexOfAny1(input, find[0])
|
||||
case 2:
|
||||
return helpers.IndexOfAny2(input, find[0], find[1])
|
||||
case 3:
|
||||
return helpers.IndexOfAny3(input, find[0], find[1], find[2])
|
||||
default:
|
||||
return helpers.IndexOfAny(input, find)
|
||||
}
|
||||
}
|
||||
|
||||
func findFixedDistanceCharLeftToRight(r *Runner, ch rune, distance int) bool {
|
||||
searchStart := r.Runtextpos + distance
|
||||
for searchStart < r.Runtextend {
|
||||
@@ -1850,6 +2036,9 @@ func (r *Runner) initMatch(textInfo *matchText) {
|
||||
if tracksize < 64 {
|
||||
tracksize = 64
|
||||
}
|
||||
if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && tracksize > limit {
|
||||
tracksize = limit
|
||||
}
|
||||
if stacksize < 32 {
|
||||
stacksize = 32
|
||||
}
|
||||
@@ -1882,9 +2071,7 @@ func (r *Runner) tidyMatch(quick bool) *Match {
|
||||
m.textpos = r.Runtextpos
|
||||
if m.matchcount[0] > 0 {
|
||||
interval := m.matches[0]
|
||||
// bytes indices aren't used so just use fast path
|
||||
m.RuneIndex = interval[0]
|
||||
m.RuneLength = interval[1]
|
||||
setCaptureFields(&m.Capture, interval[0], interval[1])
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -2056,36 +2243,6 @@ func (r *Runner) initTrackCount() {
|
||||
}
|
||||
}
|
||||
|
||||
// decodeString converts s to []rune using a shared size-classed buffer pool when
|
||||
// allowed by the regexp optimization settings. Pooled slices must be returned
|
||||
// after the runner is done with them.
|
||||
func (r *Runner) decodeString(s string) ([]rune, *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength)
|
||||
n := 0
|
||||
for _, ch := range s {
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return buf[:n], pooled
|
||||
}
|
||||
|
||||
func (r *Runner) decodeStringWithStart(s string, startAt int) (runes []rune, runeStart int, pooled *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength)
|
||||
n := 0
|
||||
runeStart = -1
|
||||
for strIdx, ch := range s {
|
||||
if startAt >= 0 && strIdx == startAt {
|
||||
runeStart = n
|
||||
}
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
if startAt >= 0 && startAt == len(s) {
|
||||
runeStart = n
|
||||
}
|
||||
return buf[:n], runeStart, pooled
|
||||
}
|
||||
|
||||
// getRunner returns a runner to use for matching re.
|
||||
func (re *Regexp) getRunner() *Runner {
|
||||
if re.runnerPool == nil {
|
||||
@@ -2097,6 +2254,7 @@ func (re *Regexp) getRunner() *Runner {
|
||||
// putRunner returns a runner to the re's pool cache.
|
||||
func (re *Regexp) putRunner(r *Runner) {
|
||||
r.Runtext = nil
|
||||
r.code = re.code
|
||||
if r.runmatch != nil {
|
||||
r.runmatch.text = nil
|
||||
}
|
||||
@@ -2128,6 +2286,12 @@ func (r *Runner) StackPop() int {
|
||||
return val
|
||||
}
|
||||
|
||||
// StackDepth returns the number of integer slots currently used by the
|
||||
// generated engine's backtracking stack.
|
||||
func (r *Runner) StackDepth() int {
|
||||
return len(r.runstack) - r.Runstackpos
|
||||
}
|
||||
|
||||
func (r *Runner) StackPush(val int) {
|
||||
// check if we need to size up stack
|
||||
r.ensureStack(1)
|
||||
|
||||
+7
-11
@@ -33,21 +33,20 @@ func (re *Regexp) Split(input string, count int) ([]string, error) {
|
||||
// iterate through the matches
|
||||
priorIndex := 0
|
||||
var retVal []string
|
||||
var txt []rune
|
||||
matched := false
|
||||
|
||||
m, err := re.FindStringMatch(input)
|
||||
|
||||
for ; m != nil && count > 0; m, err = re.FindNextMatch(m) {
|
||||
txt = m.text.runes
|
||||
// if we have an m, we don't have an err
|
||||
// append our match
|
||||
retVal = append(retVal, string(txt[priorIndex:m.RuneIndex]))
|
||||
matched = true
|
||||
start, end := matchInputSpan(m)
|
||||
retVal = append(retVal, input[priorIndex:start])
|
||||
// append any capture groups, skipping group 0
|
||||
gs := m.Groups()
|
||||
for i := 1; i < len(gs); i++ {
|
||||
retVal = append(retVal, gs[i].String())
|
||||
}
|
||||
priorIndex = m.RuneIndex + m.RuneLength
|
||||
priorIndex = end
|
||||
count--
|
||||
}
|
||||
|
||||
@@ -55,13 +54,10 @@ func (re *Regexp) Split(input string, count int) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if txt == nil {
|
||||
// we never matched, return the original string
|
||||
if !matched {
|
||||
return []string{input}, nil
|
||||
}
|
||||
|
||||
// append our remainder
|
||||
retVal = append(retVal, string(txt[priorIndex:]))
|
||||
|
||||
retVal = append(retVal, input[priorIndex:])
|
||||
return retVal, nil
|
||||
}
|
||||
|
||||
+90
@@ -39,6 +39,15 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter {
|
||||
return stringIndexPrefixesFilter(opts.LeadingPrefixes, false, minRequiredLength)
|
||||
case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight:
|
||||
return stringIndexPrefixesFilter(opts.LeadingPrefixes, true, minRequiredLength)
|
||||
case syntax.LeadingSet_LeftToRight:
|
||||
if len(opts.FixedDistanceSets) == 0 {
|
||||
return nil
|
||||
}
|
||||
set := opts.FixedDistanceSets[0]
|
||||
if set.Range == nil && (len(set.Chars) == 0 || len(set.Chars) > 5) {
|
||||
return nil
|
||||
}
|
||||
return stringFixedDistanceSetFilter(set, minRequiredLength)
|
||||
case syntax.FixedDistanceChar_LeftToRight:
|
||||
return stringFixedDistanceCharFilter(opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance, minRequiredLength)
|
||||
case syntax.FixedDistanceString_LeftToRight:
|
||||
@@ -50,6 +59,87 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter {
|
||||
}
|
||||
}
|
||||
|
||||
type asciiSetStringScanner struct {
|
||||
chars string
|
||||
first byte
|
||||
last byte
|
||||
useRange bool
|
||||
distance int
|
||||
}
|
||||
|
||||
func newASCIISetStringScanner(set syntax.FixedDistanceSet) (asciiSetStringScanner, bool) {
|
||||
if set.Negated || set.Distance < 0 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
if set.Range != nil {
|
||||
if set.Range.First < 0 || set.Range.Last > utf8.RuneSelf-1 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
return asciiSetStringScanner{
|
||||
first: byte(set.Range.First),
|
||||
last: byte(set.Range.Last),
|
||||
useRange: true,
|
||||
distance: set.Distance,
|
||||
}, true
|
||||
}
|
||||
if len(set.Chars) == 0 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
chars := make([]byte, len(set.Chars))
|
||||
for i, ch := range set.Chars {
|
||||
if ch < 0 || ch > utf8.RuneSelf-1 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
chars[i] = byte(ch)
|
||||
}
|
||||
return asciiSetStringScanner{chars: string(chars), distance: set.Distance}, true
|
||||
}
|
||||
|
||||
func stringFixedDistanceSetFilter(set syntax.FixedDistanceSet, minRequiredLength int) StringPrefixFilter {
|
||||
scanner, ok := newASCIISetStringScanner(set)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return func(input string, startAt int) (candidateByteIndex int, ok bool) {
|
||||
if !hasMinRequiredBytes(input, startAt, minRequiredLength) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
for searchAt := startAt; searchAt < len(input); {
|
||||
offset := scanner.index(input[searchAt:])
|
||||
if offset < 0 {
|
||||
return 0, false
|
||||
}
|
||||
setByteIndex := searchAt + offset
|
||||
candidateByteIndex, valid := stringFixedDistanceCandidateStart(input, startAt, setByteIndex, scanner.distance)
|
||||
if valid && hasMinRequiredBytes(input, candidateByteIndex, minRequiredLength) {
|
||||
return candidateByteIndex, true
|
||||
}
|
||||
if valid {
|
||||
return 0, false
|
||||
}
|
||||
searchAt = setByteIndex + 1
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func (s asciiSetStringScanner) index(input string) int {
|
||||
if !s.useRange {
|
||||
if len(s.chars) == 1 {
|
||||
return strings.IndexByte(input, s.chars[0])
|
||||
}
|
||||
return strings.IndexAny(input, s.chars)
|
||||
}
|
||||
for i := 0; i < len(input); i++ {
|
||||
if input[i] >= s.first && input[i] <= s.last {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func stringIndexPrefixFilter(prefix string, ignoreCase bool, minRequiredLength int) StringPrefixFilter {
|
||||
if prefix == "" {
|
||||
return nil
|
||||
|
||||
+69
-3
@@ -91,6 +91,11 @@ const (
|
||||
Setloopatomic InstOp = 45
|
||||
// Updates the bumpalong position to the current position.
|
||||
UpdateBumpalong InstOp = 46
|
||||
// Matches one Unicode extended grapheme cluster (\X).
|
||||
Grapheme InstOp = 47
|
||||
// Selects and consumes the next character using a disjoint branch table.
|
||||
// Operand 0 is an index into Code.Dispatches.
|
||||
Dispatch InstOp = 48
|
||||
|
||||
// Modifiers for alternate modes
|
||||
|
||||
@@ -105,6 +110,7 @@ type Code struct {
|
||||
Codes []int // the code
|
||||
Strings [][]rune // string table
|
||||
Sets []*CharSet //character set table
|
||||
Dispatches []DispatchTable // shared match tables for deterministic alternations
|
||||
TrackCount int // how many instructions use backtracking
|
||||
Caps map[int]int // mapping of user group numbers -> impl group slots
|
||||
Capsize int // number of impl group slots
|
||||
@@ -113,6 +119,56 @@ type Code struct {
|
||||
Anchors AnchorLoc // the set of zero-length start anchors (RegexFCD.Bol, etc)
|
||||
RightToLeft bool // true if right to left
|
||||
FindOptimizations *FindOptimizations // analyzed candidate search strategy
|
||||
QuickCodes []int // bool-only code with unobservable captures removed
|
||||
QuickDispatches []DispatchTable // lightweight tables targeting QuickCodes
|
||||
CaptureSlotInUse []bool // capture slots observable by the pattern itself during quick matches
|
||||
// LeftContextRunes is how many runes before a candidate start matching may
|
||||
// inspect. 0 means none, 1 means a single previous rune (or slack so ^/\A
|
||||
// do not see the candidate as the origin), and -1 means do not slice
|
||||
// (lookbehind or \G).
|
||||
LeftContextRunes int
|
||||
}
|
||||
|
||||
// DispatchTable maps disjoint character sets to branch indices. Larger tables
|
||||
// use a direct ASCII lookup, small tables use compact masks, and non-ASCII
|
||||
// runes inspect the full sets.
|
||||
type DispatchTable struct {
|
||||
Sets []int
|
||||
Branches []int
|
||||
ASCII *[128]uint16 // entry index + 1; zero means no branch
|
||||
ASCIIMasks []uint64 // two membership words per branch when ASCII is nil
|
||||
}
|
||||
|
||||
// captureSlotsInUse returns the capture slots whose values can affect matching.
|
||||
// Group 0 is always retained as the success marker. Ordinary captures that are
|
||||
// never referenced by the pattern may be omitted by bool-only matching APIs.
|
||||
func captureSlotsInUse(codes []int, capsize int) []bool {
|
||||
inUse := make([]bool, capsize)
|
||||
if capsize > 0 {
|
||||
inUse[0] = true
|
||||
}
|
||||
for pos := 0; pos < len(codes); {
|
||||
op := InstOp(codes[pos]) & Mask
|
||||
switch op {
|
||||
case Ref, Testref:
|
||||
capnum := codes[pos+1]
|
||||
if capnum >= 0 && capnum < len(inUse) {
|
||||
inUse[capnum] = true
|
||||
}
|
||||
case Capturemark:
|
||||
// Balancing groups both observe and mutate capture state. Keep both
|
||||
// sides live even if no later backreference refers to them.
|
||||
if codes[pos+2] != -1 {
|
||||
for _, capnum := range codes[pos+1 : pos+3] {
|
||||
if capnum >= 0 && capnum < len(inUse) {
|
||||
inUse[capnum] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pos += opcodeSize(op)
|
||||
}
|
||||
return inUse
|
||||
}
|
||||
|
||||
// PrepareCharSetASCIIBitmaps builds bounded ASCII lookup tables for compiled
|
||||
@@ -156,11 +212,11 @@ func opcodeSize(op InstOp) int {
|
||||
|
||||
switch op {
|
||||
case Nothing, Bol, Eol, Boundary, Nonboundary, ECMABoundary, NonECMABoundary, Beginning, Start, EndZ,
|
||||
End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong:
|
||||
End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong, Grapheme:
|
||||
return 1
|
||||
|
||||
case One, Notone, Multi, Ref, Testref, Goto, Nullcount, Setcount, Lazybranch, Branchmark, Lazybranchmark,
|
||||
Prune, Set:
|
||||
Prune, Set, Dispatch:
|
||||
return 2
|
||||
|
||||
case Capturemark, Branchcount, Lazybranchcount, Onerep, Notonerep, Oneloop, Notoneloop, Onelazy, Notonelazy,
|
||||
@@ -187,7 +243,7 @@ var codeStr = []string{
|
||||
"Prune", "Stop",
|
||||
"ECMABoundary", "NonECMABoundary",
|
||||
"Oneloopatomic", "Notoneloopatomic", "Setloopatomic",
|
||||
"Bumpalong",
|
||||
"Bumpalong", "Grapheme", "Dispatch",
|
||||
}
|
||||
|
||||
func operatorDescription(op InstOp) string {
|
||||
@@ -234,6 +290,16 @@ func (c *Code) OpcodeDescription(offset int) string {
|
||||
buf.WriteString("Set = ")
|
||||
buf.WriteString(c.Sets[c.Codes[offset+1]].String())
|
||||
|
||||
case Dispatch:
|
||||
tableIndex := c.Codes[offset+1]
|
||||
fmt.Fprintf(buf, "Table = %d", tableIndex)
|
||||
if tableIndex >= 0 && tableIndex < len(c.Dispatches) {
|
||||
table := &c.Dispatches[tableIndex]
|
||||
for i, setIndex := range table.Sets {
|
||||
fmt.Fprintf(buf, ", %s -> %d", c.Sets[setIndex].String(), table.Branches[i])
|
||||
}
|
||||
}
|
||||
|
||||
case Multi:
|
||||
fmt.Fprintf(buf, "String = %s", string(c.Strings[c.Codes[offset+1]]))
|
||||
|
||||
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
package syntax
|
||||
|
||||
import "unicode"
|
||||
|
||||
type graphemeBreakClass uint8
|
||||
|
||||
const (
|
||||
graphemeOther graphemeBreakClass = iota
|
||||
graphemeCR
|
||||
graphemeLF
|
||||
graphemeControl
|
||||
graphemeExtend
|
||||
graphemeZWJ
|
||||
graphemeRegionalIndicator
|
||||
graphemePrepend
|
||||
graphemeSpacingMark
|
||||
graphemeL
|
||||
graphemeV
|
||||
graphemeT
|
||||
graphemeLV
|
||||
graphemeLVT
|
||||
)
|
||||
|
||||
type indicConjunctBreakClass uint8
|
||||
|
||||
const (
|
||||
indicNone indicConjunctBreakClass = iota
|
||||
indicConsonant
|
||||
indicExtend
|
||||
indicLinker
|
||||
)
|
||||
|
||||
type graphemeProperties struct {
|
||||
breakClass graphemeBreakClass
|
||||
indicClass indicConjunctBreakClass
|
||||
extendedPictographic bool
|
||||
}
|
||||
|
||||
// NextGraphemeClusterBoundary returns the first extended-grapheme boundary
|
||||
// after start, or -1 when start is not a valid input position. It implements
|
||||
// the Unicode 17 rules from UAX #29 with a small state machine, avoiding the
|
||||
// backtracking and repeated character-class probes of an equivalent regexp.
|
||||
func NextGraphemeClusterBoundary(text []rune, start int) int {
|
||||
if start < 0 || start >= len(text) {
|
||||
return -1
|
||||
}
|
||||
|
||||
previous := graphemePropertiesFor(text[start])
|
||||
state := graphemeForwardState{}
|
||||
state.consume(previous)
|
||||
|
||||
for pos := start + 1; pos < len(text); pos++ {
|
||||
current := graphemePropertiesFor(text[pos])
|
||||
if isGraphemeBoundaryForward(previous.breakClass, current, state) {
|
||||
return pos
|
||||
}
|
||||
state.consume(current)
|
||||
previous = current
|
||||
}
|
||||
return len(text)
|
||||
}
|
||||
|
||||
// PreviousGraphemeClusterBoundary returns the first extended-grapheme boundary
|
||||
// before end, or -1 when end is not a valid input position.
|
||||
func PreviousGraphemeClusterBoundary(text []rune, end int) int {
|
||||
if end <= 0 || end > len(text) {
|
||||
return -1
|
||||
}
|
||||
for pos := end - 1; pos > 0; pos-- {
|
||||
if isGraphemeBoundary(text, pos) {
|
||||
return pos
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type graphemeForwardState struct {
|
||||
regionalIndicatorCount int
|
||||
extendedPictographicRun bool
|
||||
zwjAfterPictographic bool
|
||||
indicConsonantRun bool
|
||||
indicLinkerSeen bool
|
||||
}
|
||||
|
||||
func (s *graphemeForwardState) consume(properties graphemeProperties) {
|
||||
if properties.breakClass == graphemeRegionalIndicator {
|
||||
s.regionalIndicatorCount++
|
||||
} else {
|
||||
s.regionalIndicatorCount = 0
|
||||
}
|
||||
|
||||
if properties.breakClass == graphemeZWJ {
|
||||
s.zwjAfterPictographic = s.extendedPictographicRun
|
||||
} else {
|
||||
s.zwjAfterPictographic = false
|
||||
}
|
||||
if properties.extendedPictographic {
|
||||
s.extendedPictographicRun = true
|
||||
} else if properties.breakClass != graphemeExtend {
|
||||
s.extendedPictographicRun = false
|
||||
}
|
||||
|
||||
switch properties.indicClass {
|
||||
case indicConsonant:
|
||||
s.indicConsonantRun = true
|
||||
s.indicLinkerSeen = false
|
||||
case indicExtend:
|
||||
// Extend preserves a preceding consonant/linker run.
|
||||
case indicLinker:
|
||||
if s.indicConsonantRun {
|
||||
s.indicLinkerSeen = true
|
||||
}
|
||||
default:
|
||||
s.indicConsonantRun = false
|
||||
s.indicLinkerSeen = false
|
||||
}
|
||||
}
|
||||
|
||||
func isGraphemeBoundaryForward(previous graphemeBreakClass, current graphemeProperties, state graphemeForwardState) bool {
|
||||
if previous == graphemeCR && current.breakClass == graphemeLF { // GB3
|
||||
return false
|
||||
}
|
||||
if isGraphemeControl(previous) || isGraphemeControl(current.breakClass) { // GB4, GB5
|
||||
return true
|
||||
}
|
||||
if hangulNoBreak(previous, current.breakClass) { // GB6, GB7, GB8
|
||||
return false
|
||||
}
|
||||
if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a
|
||||
return false
|
||||
}
|
||||
if previous == graphemePrepend { // GB9b
|
||||
return false
|
||||
}
|
||||
if current.indicClass == indicConsonant && state.indicConsonantRun && state.indicLinkerSeen { // GB9c
|
||||
return false
|
||||
}
|
||||
if current.extendedPictographic && state.zwjAfterPictographic { // GB11
|
||||
return false
|
||||
}
|
||||
if previous == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator && state.regionalIndicatorCount%2 == 1 { // GB12, GB13
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isGraphemeBoundary(text []rune, pos int) bool {
|
||||
previous := graphemePropertiesFor(text[pos-1])
|
||||
current := graphemePropertiesFor(text[pos])
|
||||
if previous.breakClass == graphemeCR && current.breakClass == graphemeLF { // GB3
|
||||
return false
|
||||
}
|
||||
if isGraphemeControl(previous.breakClass) || isGraphemeControl(current.breakClass) { // GB4, GB5
|
||||
return true
|
||||
}
|
||||
if hangulNoBreak(previous.breakClass, current.breakClass) { // GB6, GB7, GB8
|
||||
return false
|
||||
}
|
||||
if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a
|
||||
return false
|
||||
}
|
||||
if previous.breakClass == graphemePrepend { // GB9b
|
||||
return false
|
||||
}
|
||||
if current.indicClass == indicConsonant && hasIndicConjunctBefore(text, pos) { // GB9c
|
||||
return false
|
||||
}
|
||||
if current.extendedPictographic && hasExtendedPictographicZWJBefore(text, pos) { // GB11
|
||||
return false
|
||||
}
|
||||
if previous.breakClass == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator { // GB12, GB13
|
||||
count := 0
|
||||
for i := pos - 1; i >= 0 && graphemeClass(text[i]) == graphemeRegionalIndicator; i-- {
|
||||
count++
|
||||
}
|
||||
return count%2 == 0
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func hasIndicConjunctBefore(text []rune, pos int) bool {
|
||||
linkerSeen := false
|
||||
for i := pos - 1; i >= 0; i-- {
|
||||
switch indicClass(text[i]) {
|
||||
case indicExtend:
|
||||
continue
|
||||
case indicLinker:
|
||||
linkerSeen = true
|
||||
continue
|
||||
case indicConsonant:
|
||||
return linkerSeen
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasExtendedPictographicZWJBefore(text []rune, pos int) bool {
|
||||
i := pos - 1
|
||||
if i < 0 || graphemeClass(text[i]) != graphemeZWJ {
|
||||
return false
|
||||
}
|
||||
for i--; i >= 0 && graphemeClass(text[i]) == graphemeExtend; i-- {
|
||||
}
|
||||
return i >= 0 && unicode.Is(unicodeAliasExtended_Pictographic, text[i])
|
||||
}
|
||||
|
||||
func isGraphemeControl(class graphemeBreakClass) bool {
|
||||
return class == graphemeCR || class == graphemeLF || class == graphemeControl
|
||||
}
|
||||
|
||||
func hangulNoBreak(previous, current graphemeBreakClass) bool {
|
||||
return previous == graphemeL && (current == graphemeL || current == graphemeV || current == graphemeLV || current == graphemeLVT) ||
|
||||
(previous == graphemeLV || previous == graphemeV) && (current == graphemeV || current == graphemeT) ||
|
||||
(previous == graphemeLVT || previous == graphemeT) && current == graphemeT
|
||||
}
|
||||
|
||||
func indicClass(ch rune) indicConjunctBreakClass {
|
||||
if ch < 0x300 {
|
||||
return indicNone
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Consonant, ch) {
|
||||
return indicConsonant
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Linker, ch) {
|
||||
return indicLinker
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Extend, ch) {
|
||||
return indicExtend
|
||||
}
|
||||
return indicNone
|
||||
}
|
||||
|
||||
func graphemePropertiesFor(ch rune) graphemeProperties {
|
||||
if ch <= unicode.MaxASCII {
|
||||
return graphemeProperties{breakClass: graphemeClass(ch)}
|
||||
}
|
||||
return graphemeProperties{
|
||||
breakClass: graphemeClass(ch),
|
||||
indicClass: indicClass(ch),
|
||||
extendedPictographic: ch >= 0xA9 && unicode.Is(unicodeAliasExtended_Pictographic, ch),
|
||||
}
|
||||
}
|
||||
|
||||
func graphemeClass(ch rune) graphemeBreakClass {
|
||||
if ch <= unicode.MaxASCII {
|
||||
switch ch {
|
||||
case '\r':
|
||||
return graphemeCR
|
||||
case '\n':
|
||||
return graphemeLF
|
||||
default:
|
||||
if ch < ' ' || ch == 0x7F {
|
||||
return graphemeControl
|
||||
}
|
||||
return graphemeOther
|
||||
}
|
||||
}
|
||||
if ch == 0x200D {
|
||||
return graphemeZWJ
|
||||
}
|
||||
if ch >= 0x1F1E6 && ch <= 0x1F1FF {
|
||||
return graphemeRegionalIndicator
|
||||
}
|
||||
if ch >= 0xAC00 && ch <= 0xD7A3 {
|
||||
if (ch-0xAC00)%28 == 0 {
|
||||
return graphemeLV
|
||||
}
|
||||
return graphemeLVT
|
||||
}
|
||||
if ch >= 0x1100 && ch <= 0x115F || ch >= 0xA960 && ch <= 0xA97C {
|
||||
return graphemeL
|
||||
}
|
||||
if ch >= 0x1160 && ch <= 0x11A7 || ch >= 0xD7B0 && ch <= 0xD7C6 {
|
||||
return graphemeV
|
||||
}
|
||||
if ch >= 0x11A8 && ch <= 0x11FF || ch >= 0xD7CB && ch <= 0xD7FB {
|
||||
return graphemeT
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Control, ch) {
|
||||
return graphemeControl
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Extend, ch) {
|
||||
return graphemeExtend
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Prepend, ch) {
|
||||
return graphemePrepend
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_SpacingMark, ch) {
|
||||
return graphemeSpacingMark
|
||||
}
|
||||
return graphemeOther
|
||||
}
|
||||
+26
-24
@@ -21,10 +21,11 @@ type FindOptimizations struct {
|
||||
LeadingPrefixesRunes [][]rune
|
||||
//LeadingStrings *helpers.StringSearchValues
|
||||
|
||||
FixedDistanceLiteral FixedDistanceLiteral
|
||||
FixedDistanceSets []FixedDistanceSet
|
||||
LiteralAfterLoop *LiteralAfterLoop
|
||||
LandmarkChain *RequiredLandmarkChain
|
||||
FixedDistanceLiteral FixedDistanceLiteral
|
||||
FixedDistanceSets []FixedDistanceSet
|
||||
LiteralAfterLoop *LiteralAfterLoop
|
||||
LandmarkChain *RequiredLandmarkChain
|
||||
LeadingPrefixFirstRunes []rune
|
||||
}
|
||||
|
||||
type LiteralAfterLoop struct {
|
||||
@@ -460,19 +461,23 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar
|
||||
|
||||
// We're now left-to-right only and looking for multiple prefixes and/or sets.
|
||||
|
||||
// If there are multiple leading strings, we can search for any of them.
|
||||
// this works in the interpreter, but we avoid it due to additional cost during construction
|
||||
|
||||
// Multiple leading strings let the finder jump between candidate prefixes.
|
||||
// Case-sensitive prefixes are cheap to collect and help the interpreter.
|
||||
// Case-insensitive prefix explosion is still limited to code generation.
|
||||
if prefixes := findPrefixes(root, false); len(prefixes) > 1 {
|
||||
f.LeadingPrefixes = prefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(prefixes)
|
||||
f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes)
|
||||
f.FindMode = LeadingStrings_LeftToRight
|
||||
return f
|
||||
}
|
||||
if !interpreter {
|
||||
ciPrefixes := findPrefixes(root, true)
|
||||
if len(ciPrefixes) > 1 {
|
||||
f.LeadingPrefixes = ciPrefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(ciPrefixes)
|
||||
f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes)
|
||||
f.FindMode = LeadingStrings_OrdinalIgnoreCase_LeftToRight
|
||||
/*SYSTEM_TEXT_REGULAREXPRESSIONS
|
||||
if usesRfoTryFind {
|
||||
f.LeadingStrings = helpers.NewSearchValues(f.LeadingPrefixes, true)
|
||||
}*/
|
||||
return f
|
||||
}
|
||||
}
|
||||
@@ -515,19 +520,6 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar
|
||||
// In some searches, we may use multiple sets, so we want the subsequent ones to also be the efficiency runners-up.
|
||||
slices.SortFunc(fixedDistanceSets, compareFixedDistanceSetsByQuality)
|
||||
|
||||
// If the best fixed-distance set is composed of high-frequency characters, IndexOfAny on
|
||||
// those characters is likely to match too many positions. Prefer a case-sensitive
|
||||
// multi-prefix search when one is available.
|
||||
if !interpreter && !mayContainCaseInsensitiveMatching(root) && hasHighFrequencyChars(fixedDistanceSets[0]) {
|
||||
caseSensitivePrefixes := findPrefixes(root, false)
|
||||
if len(caseSensitivePrefixes) > 1 {
|
||||
f.LeadingPrefixes = caseSensitivePrefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(caseSensitivePrefixes)
|
||||
f.FindMode = LeadingStrings_LeftToRight
|
||||
return f
|
||||
}
|
||||
}
|
||||
|
||||
// If there is no literal after the loop, use whatever set we got.
|
||||
// If there is a literal after the loop, consider it to be better than a negated set and better than a set with many characters.
|
||||
if literalAfterLoop == nil || (len(fixedDistanceSets[0].Chars) > 0 && !fixedDistanceSets[0].Negated) {
|
||||
@@ -587,6 +579,16 @@ func toRunePrefixes(prefixes []string) [][]rune {
|
||||
return runes
|
||||
}
|
||||
|
||||
func leadingPrefixFirstRunes(prefixes [][]rune) []rune {
|
||||
first := make([]rune, 0, len(prefixes))
|
||||
for _, prefix := range prefixes {
|
||||
if len(prefix) > 0 && !slices.Contains(first, prefix[0]) {
|
||||
first = append(first, prefix[0])
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func getFindMode(rtl bool, t NodeType) FindNextStartingPositionMode {
|
||||
if rtl {
|
||||
switch t {
|
||||
|
||||
+168
-13
@@ -636,11 +636,16 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
}
|
||||
|
||||
case '\\':
|
||||
quoted := !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q'
|
||||
n, err := p.scanBackslash(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.addUnitNode(n)
|
||||
if quoted {
|
||||
p.addQuotedUnit(n)
|
||||
} else {
|
||||
p.addUnitNode(n)
|
||||
}
|
||||
|
||||
case '^':
|
||||
if p.useOptionM() {
|
||||
@@ -702,7 +707,7 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
// Handle quantifiers
|
||||
for p.unit != nil {
|
||||
var min, max int
|
||||
var lazy bool
|
||||
var lazy, possessive bool
|
||||
|
||||
switch ch {
|
||||
case '*':
|
||||
@@ -753,18 +758,19 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if p.charsRight() == 0 || p.rightChar(0) != '?' {
|
||||
lazy = false
|
||||
} else {
|
||||
if p.charsRight() > 0 && p.rightChar(0) == '?' {
|
||||
p.moveRight(1)
|
||||
lazy = true
|
||||
} else if p.charsRight() > 0 && p.rightChar(0) == '+' && !p.useOptionE() {
|
||||
p.moveRight(1)
|
||||
possessive = true
|
||||
}
|
||||
|
||||
if min > max {
|
||||
return nil, p.getErr(ErrInvalidRepeatSize)
|
||||
}
|
||||
|
||||
p.addConcatenate3(lazy, min, max)
|
||||
p.addConcatenate3(lazy, possessive, min, max)
|
||||
}
|
||||
|
||||
ContinueOuterScan:
|
||||
@@ -1280,6 +1286,37 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
}
|
||||
|
||||
switch ch := p.rightChar(0); ch {
|
||||
case 'Q':
|
||||
if p.useOptionE() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
quoted := p.scanQuoted()
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newRegexNodeStr(NtMulti, p.options&^IgnoreCase, quoted), nil
|
||||
|
||||
case 'R':
|
||||
if p.useOptionE() || p.useRE2() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newUnicodeNewlineNode(p.options), nil
|
||||
|
||||
case 'X':
|
||||
if p.useOptionE() || p.useRE2() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newRegexNode(NtGrapheme, p.options), nil
|
||||
|
||||
case 'b', 'B', 'A', 'G', 'Z', 'z':
|
||||
p.moveRight(1)
|
||||
return newRegexNode(p.typeFromCode(ch), p.options), nil
|
||||
@@ -1354,6 +1391,58 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// newUnicodeNewlineNode constructs \R from existing general-purpose nodes.
|
||||
// Its branches have disjoint starting sets, and the optional LF is atomic, so
|
||||
// reduction can remove the outer atomic wrapper without changing semantics.
|
||||
func newUnicodeNewlineNode(options RegexOptions) *RegexNode {
|
||||
options &^= IgnoreCase
|
||||
|
||||
lead, optional := '\r', '\n'
|
||||
if options&RightToLeft != 0 {
|
||||
// Stored concatenations are already in execution order. In RTL, consume
|
||||
// LF first and then the optional preceding CR.
|
||||
lead, optional = '\n', '\r'
|
||||
}
|
||||
|
||||
newlines := &CharSet{}
|
||||
for _, ch := range []rune{'\n', '\v', '\f', '\u0085', '\u2028', '\u2029'} {
|
||||
if ch != lead {
|
||||
newlines.addChar(ch)
|
||||
}
|
||||
}
|
||||
if '\r' != lead {
|
||||
newlines.addChar('\r')
|
||||
}
|
||||
|
||||
crlf := newRegexNode(NtConcatenate, options)
|
||||
crlf.addChild(newRegexNodeCh(NtOne, options, lead))
|
||||
crlf.addChild(newRegexNodeCh(NtOne, options, optional).makeQuantifier(false, 0, 1))
|
||||
|
||||
alternate := newRegexNode(NtAlternate, options)
|
||||
alternate.addChild(newRegexNodeSet(NtSet, options, newlines))
|
||||
alternate.addChild(crlf)
|
||||
|
||||
atomic := newRegexNode(NtAtomic, options)
|
||||
atomic.addChild(alternate)
|
||||
return atomic
|
||||
}
|
||||
|
||||
// scanQuoted scans the literal text after \Q through the next \E, or through
|
||||
// the end of the pattern when there is no terminator.
|
||||
func (p *parser) scanQuoted() []rune {
|
||||
start := p.textpos()
|
||||
var quoted []rune
|
||||
for p.charsRight() > 0 {
|
||||
if p.rightChar(0) == '\\' && p.charsRight() > 1 && p.rightChar(1) == 'E' {
|
||||
quoted = append(quoted, p.pattern[start:p.textpos()]...)
|
||||
p.moveRight(2)
|
||||
return quoted
|
||||
}
|
||||
p.moveRight(1)
|
||||
}
|
||||
return append(quoted, p.pattern[start:p.textpos()]...)
|
||||
}
|
||||
|
||||
// Scans \-style backreferences and character escapes
|
||||
func (p *parser) scanBasicBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
if p.charsRight() == 0 {
|
||||
@@ -1684,6 +1773,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
inRange := false
|
||||
firstChar := true
|
||||
closed := false
|
||||
var quoted []rune
|
||||
|
||||
var cc *CharSet
|
||||
if !scanOnly {
|
||||
@@ -1697,11 +1787,33 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
}
|
||||
}
|
||||
|
||||
for ; p.charsRight() > 0; firstChar = false {
|
||||
for ; p.charsRight() > 0 || len(quoted) > 0; firstChar = false {
|
||||
fTranslatedChar := false
|
||||
ch = p.moveRightGetChar()
|
||||
for {
|
||||
if len(quoted) > 0 {
|
||||
ch = quoted[0]
|
||||
quoted = quoted[1:]
|
||||
fTranslatedChar = true
|
||||
break
|
||||
}
|
||||
ch = p.moveRightGetChar()
|
||||
if ch == '\\' && !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q' {
|
||||
p.moveRight(1)
|
||||
quoted = p.scanQuoted()
|
||||
if len(quoted) == 0 {
|
||||
if p.charsRight() == 0 {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if ch == ']' {
|
||||
if !firstChar {
|
||||
if fTranslatedChar {
|
||||
// A quoted closing bracket is an ordinary class member.
|
||||
} else if !firstChar {
|
||||
closed = true
|
||||
break
|
||||
} else if p.useOptionE() {
|
||||
@@ -1712,7 +1824,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
break
|
||||
}
|
||||
|
||||
} else if ch == '\\' && p.charsRight() > 0 {
|
||||
} else if ch == '\\' && !fTranslatedChar && p.charsRight() > 0 {
|
||||
switch ch = p.moveRightGetChar(); ch {
|
||||
case 'D', 'd':
|
||||
if !scanOnly {
|
||||
@@ -1815,7 +1927,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
}
|
||||
fTranslatedChar = true
|
||||
}
|
||||
} else if ch == '[' {
|
||||
} else if ch == '[' && !fTranslatedChar {
|
||||
// This is code for Posix style properties - [:Ll:] or [:IsTibetan:].
|
||||
// It currently doesn't do anything other than skip the whole thing!
|
||||
if p.charsRight() > 0 && p.rightChar(0) == ':' && !inRange {
|
||||
@@ -2291,11 +2403,54 @@ func (p *parser) addConcatenate() {
|
||||
}
|
||||
|
||||
// Finish the current quantifiable (when a quantifier is found)
|
||||
func (p *parser) addConcatenate3(lazy bool, min, max int) {
|
||||
p.concatenation.addChild(p.unit.makeQuantifier(lazy, min, max))
|
||||
func (p *parser) addConcatenate3(lazy, possessive bool, min, max int) {
|
||||
node := p.unit.makeQuantifier(lazy, min, max)
|
||||
if possessive {
|
||||
atomic := newRegexNode(NtAtomic, p.options)
|
||||
atomic.addChild(node)
|
||||
node = atomic
|
||||
}
|
||||
p.concatenation.addChild(node)
|
||||
p.unit = nil
|
||||
}
|
||||
|
||||
// addQuotedUnit adds the literal characters scanned by \Q...\E. All but the
|
||||
// final character are completed immediately so a following quantifier applies
|
||||
// to the final literal character, just as if each character had been escaped.
|
||||
func (p *parser) addQuotedUnit(node *RegexNode) {
|
||||
quoted := node.Str
|
||||
if len(quoted) == 0 {
|
||||
p.restoreLastUnit()
|
||||
if p.unit == nil {
|
||||
p.unit = newRegexNode(NtEmpty, p.options)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, ch := range quoted[:len(quoted)-1] {
|
||||
p.concatenation.addChild(newRegexNodeCh(NtOne, p.options, ch))
|
||||
}
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, quoted[len(quoted)-1])
|
||||
}
|
||||
|
||||
// restoreLastUnit makes an immediately preceding literal quantifiable again
|
||||
// when an empty \Q\E appears between it and its quantifier.
|
||||
func (p *parser) restoreLastUnit() {
|
||||
if len(p.concatenation.Children) == 0 {
|
||||
return
|
||||
}
|
||||
last := len(p.concatenation.Children) - 1
|
||||
node := p.concatenation.Children[last]
|
||||
p.concatenation.Children = p.concatenation.Children[:last]
|
||||
node.Parent = nil
|
||||
if node.T == NtMulti && len(node.Str) > 1 {
|
||||
prefix := append([]rune(nil), node.Str[:len(node.Str)-1]...)
|
||||
p.concatenation.addChild(newRegexNodeStr(NtMulti, node.Options, prefix))
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, node.Str[len(node.Str)-1])
|
||||
return
|
||||
}
|
||||
p.unit = node
|
||||
}
|
||||
|
||||
// Sets the current unit to a single char node
|
||||
func (p *parser) addUnitOne(ch rune) {
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, ch)
|
||||
|
||||
+3
@@ -245,6 +245,9 @@ func (s *regexFcd) calculateFC(nt NodeType, node *RegexNode, CurIndex int) {
|
||||
case NtSetloop, NtSetlazy, NtSetloopatomic:
|
||||
s.pushFC(regexFc{cc: node.Set.Copy(), nullable: node.M == 0, caseInsensitive: ci})
|
||||
|
||||
case NtGrapheme:
|
||||
s.pushFC(regexFc{cc: *AnyClass(), nullable: false})
|
||||
|
||||
case NtRef:
|
||||
s.pushFC(regexFc{cc: *AnyClass(), nullable: true, caseInsensitive: false})
|
||||
|
||||
|
||||
+10
-45
@@ -123,6 +123,11 @@ func tryFindFirstCharClass(node *RegexNode, ccIn **CharSet) int {
|
||||
}
|
||||
return 0
|
||||
|
||||
case NtGrapheme:
|
||||
// Every rune can begin a grapheme, so there is no useful candidate
|
||||
// restriction to derive.
|
||||
return 0
|
||||
|
||||
// Zero-width elements. These don't contribute to the starting set, so return null to indicate a caller
|
||||
// should keep looking past them.
|
||||
case NtEmpty, NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary,
|
||||
@@ -532,6 +537,9 @@ func findPrefixesCore(node *RegexNode, res *[]*bytes.Buffer, ignoreCase bool) bo
|
||||
// that comprise the set. For case-insensitive, we need the set to be two ASCII letters that case fold to the same thing.
|
||||
// As with One and loops, set loops are handled the same as sets up to the min iteration limit.
|
||||
case NtSet, NtSetloop, NtSetlazy, NtSetloopatomic:
|
||||
if node.Set == nil || node.Set.IsNegated() {
|
||||
return false
|
||||
}
|
||||
|
||||
setChars := node.Set.GetSetChars(maxPrefixes)
|
||||
|
||||
@@ -895,10 +903,11 @@ func tryFindRawFixedSets(node *RegexNode, res *[]FixedDistanceSet, distance *int
|
||||
combined[fixedSet.Distance] = v
|
||||
}
|
||||
} else {
|
||||
setCopy := fixedSet.Set.Copy()
|
||||
combined[fixedSet.Distance] = struct {
|
||||
Set *CharSet
|
||||
Count int
|
||||
}{Set: fixedSet.Set, Count: 1}
|
||||
}{Set: &setCopy, Count: 1}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1034,50 +1043,6 @@ func sumFrequencies(chars []rune) float32 {
|
||||
return sum
|
||||
}
|
||||
|
||||
func hasHighFrequencyChars(set FixedDistanceSet) bool {
|
||||
if set.Negated {
|
||||
return true
|
||||
}
|
||||
|
||||
// Sets without extracted chars can't be frequency-analyzed.
|
||||
// Single-char sets use IndexOf, which is a strong filter regardless of frequency.
|
||||
if len(set.Chars) <= 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
totalFrequency := sumFrequencies(set.Chars)
|
||||
|
||||
// If the average frequency of the set's chars exceeds this threshold, the
|
||||
// characters are common enough that a multi-string search may be a better filter.
|
||||
const highFrequencyThreshold = 0.6
|
||||
return totalFrequency >= highFrequencyThreshold*float32(len(set.Chars))
|
||||
}
|
||||
|
||||
func mayContainCaseInsensitiveMatching(node *RegexNode) bool {
|
||||
if node.Options&IgnoreCase != 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
if node.Set != nil {
|
||||
chars := node.Set.GetSetChars(maxPrefixes)
|
||||
for _, ch := range chars {
|
||||
if participatesInCaseConversion(ch) &&
|
||||
slices.Contains(chars, unicode.ToLower(ch)) &&
|
||||
slices.Contains(chars, unicode.ToUpper(ch)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, child := range node.Children {
|
||||
if mayContainCaseInsensitiveMatching(child) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Percent occurrences in source text (100 * char count / total count)
|
||||
var frequency = []float32{
|
||||
0.000 /* '\x00' */, 0.000 /* '\x01' */, 0.000 /* '\x02' */, 0.000 /* '\x03' */, 0.000 /* '\x04' */, 0.000 /* '\x05' */, 0.000 /* '\x06' */, 0.000, /* '\x07' */
|
||||
|
||||
+119
-1
@@ -128,6 +128,8 @@ const (
|
||||
NtSetloopatomic NodeType = 45
|
||||
// Updates the bumpalong position to the current position.
|
||||
NtUpdateBumpalong NodeType = 46
|
||||
// Matches one Unicode extended grapheme cluster (\X).
|
||||
NtGrapheme NodeType = 47
|
||||
)
|
||||
|
||||
func newRegexNode(t NodeType, opt RegexOptions) *RegexNode {
|
||||
@@ -693,10 +695,117 @@ func (n *RegexNode) reduceAtomic() *RegexNode {
|
||||
// For everything else, try to reduce ending backtracking of the last contained expression.
|
||||
default:
|
||||
child.eliminateEndingBacktracking()
|
||||
if child.T == NtAlternate && child.hasDisjointStartingSets() && child.hasOnlyIntrinsicallyAtomicBranches() {
|
||||
return child
|
||||
}
|
||||
return atomic
|
||||
}
|
||||
}
|
||||
|
||||
// hasDisjointStartingSets reports whether every branch is non-nullable and
|
||||
// begins with a character set that cannot overlap any other branch. In that
|
||||
// case, once one branch has matched, no sibling branch could match the same
|
||||
// starting position.
|
||||
func (n *RegexNode) hasDisjointStartingSets() bool {
|
||||
_, ok := n.disjointStartingSets()
|
||||
return ok
|
||||
}
|
||||
|
||||
// disjointStartingSets returns the non-nullable first-character set for each
|
||||
// branch when all of those sets are pairwise disjoint.
|
||||
func (n *RegexNode) disjointStartingSets() ([]*CharSet, bool) {
|
||||
sets := make([]*CharSet, 0, len(n.Children))
|
||||
for _, branch := range n.Children {
|
||||
var set *CharSet
|
||||
if tryFindFirstCharClass(branch, &set) != 1 || set == nil {
|
||||
return nil, false
|
||||
}
|
||||
for _, previous := range sets {
|
||||
if set.MayOverlap(previous) {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
sets = append(sets, set)
|
||||
}
|
||||
return sets, true
|
||||
}
|
||||
|
||||
// disjointAtomicBranchSets returns the first-character sets when an
|
||||
// alternation can select a branch without leaving a backtracking choice.
|
||||
func (n *RegexNode) disjointAtomicBranchSets() ([]*CharSet, bool) {
|
||||
if n.T != NtAlternate || !n.hasOnlyIntrinsicallyAtomicBranches() {
|
||||
return nil, false
|
||||
}
|
||||
return n.disjointStartingSets()
|
||||
}
|
||||
|
||||
// dispatchCandidates returns disjoint branch sets and the leading nodes that
|
||||
// can be folded into a consuming Dispatch instruction.
|
||||
func (n *RegexNode) dispatchCandidates() ([]*CharSet, []*RegexNode, []bool, bool) {
|
||||
if len(n.Children) >= 1<<16 {
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
sets, ok := n.disjointAtomicBranchSets()
|
||||
if !ok {
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
|
||||
leaders := make([]*RegexNode, len(n.Children))
|
||||
complete := make([]bool, len(n.Children))
|
||||
for i := range leaders {
|
||||
branch := n.Children[i]
|
||||
leader := branch
|
||||
if leader.T == NtConcatenate && len(leader.Children) > 0 {
|
||||
leader = leader.Children[0]
|
||||
}
|
||||
switch leader.T {
|
||||
case NtOne, NtNotone, NtSet, NtMulti:
|
||||
leaders[i] = leader
|
||||
default:
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
complete[i] = leader == branch && (leader.T != NtMulti || len(leader.Str) == 1)
|
||||
}
|
||||
return sets, leaders, complete, true
|
||||
}
|
||||
|
||||
func (n *RegexNode) hasOnlyIntrinsicallyAtomicBranches() bool {
|
||||
for _, branch := range n.Children {
|
||||
if !branch.isIntrinsicallyAtomic() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// isIntrinsicallyAtomic reports whether a successful match of this node has no
|
||||
// alternative input-consuming path to explore if something later fails.
|
||||
func (n *RegexNode) isIntrinsicallyAtomic() bool {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet, NtMulti, NtRef, NtGrapheme,
|
||||
NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary,
|
||||
NtBeginning, NtStart, NtEndZ, NtEnd, NtNothing, NtEmpty, NtUpdateBumpalong,
|
||||
NtOneloopatomic, NtNotoneloopatomic, NtSetloopatomic,
|
||||
NtPosLook, NtNegLook, NtAtomic:
|
||||
return true
|
||||
|
||||
case NtOneloop, NtNotoneloop, NtSetloop, NtOnelazy, NtNotonelazy, NtSetlazy:
|
||||
return n.M == n.N
|
||||
|
||||
case NtCapture, NtGroup:
|
||||
return len(n.Children) == 1 && n.Children[0].isIntrinsicallyAtomic()
|
||||
|
||||
case NtConcatenate:
|
||||
for _, child := range n.Children {
|
||||
if !child.isIntrinsicallyAtomic() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (n *RegexNode) makeLoopAtomic() {
|
||||
|
||||
switch n.T {
|
||||
@@ -1680,6 +1789,12 @@ func (n *RegexNode) reduceConcatenationWithAdjacentStrings() {
|
||||
// Nested repeaters just get multiplied with each other if they're not
|
||||
// too lumpy
|
||||
func (n *RegexNode) reduceRep() *RegexNode {
|
||||
// Repeating a literal empty expression has no observable effect. In addition to
|
||||
// being unnecessary, retaining the loop would require one backtracking frame per
|
||||
// mandatory iteration even though no input is consumed.
|
||||
if len(n.Children) == 1 && n.Children[0].T == NtEmpty {
|
||||
return n.Children[0]
|
||||
}
|
||||
|
||||
u := n
|
||||
t := n.T
|
||||
@@ -1865,7 +1980,7 @@ func (n *RegexNode) makeQuantifier(lazy bool, min, max int) *RegexNode {
|
||||
// If the result is 0, there is no minimum we can enforce.
|
||||
func (n *RegexNode) ComputeMinLength() int {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet:
|
||||
case NtOne, NtNotone, NtSet, NtGrapheme:
|
||||
// single char
|
||||
return 1
|
||||
case NtMulti:
|
||||
@@ -1937,6 +2052,8 @@ func (n *RegexNode) computeMaxLength() int {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet:
|
||||
return 1
|
||||
case NtGrapheme:
|
||||
return -1
|
||||
case NtMulti:
|
||||
return len(n.Str)
|
||||
case NtNotonelazy, NtNotoneloop, NtNotoneloopatomic,
|
||||
@@ -2049,6 +2166,7 @@ var typeStr = []string{
|
||||
"ECMABoundary", "NonECMABoundary",
|
||||
"OneloopAtomic", "NotoneloopAtomic", "SetloopAtomic",
|
||||
"UpdateBumpalong",
|
||||
"Grapheme",
|
||||
}
|
||||
|
||||
func (n *RegexNode) Description() string {
|
||||
|
||||
+536
@@ -41,6 +41,9 @@ var unicodeAliasCategories = map[string]*unicode.RangeTable{
|
||||
"Grapheme_Cluster_Break=T": unicodeAliasGrapheme_Cluster_Break_T,
|
||||
"Grapheme_Cluster_Break=V": unicodeAliasGrapheme_Cluster_Break_V,
|
||||
"Grapheme_Cluster_Break=ZWJ": unicodeAliasGrapheme_Cluster_Break_ZWJ,
|
||||
"Indic_Conjunct_Break=Consonant": unicodeAliasIndic_Conjunct_Break_Consonant,
|
||||
"Indic_Conjunct_Break=Extend": unicodeAliasIndic_Conjunct_Break_Extend,
|
||||
"Indic_Conjunct_Break=Linker": unicodeAliasIndic_Conjunct_Break_Linker,
|
||||
"Math": unicodeAliasMath,
|
||||
"Sentence_Break=ATerm": unicodeAliasSentence_Break_ATerm,
|
||||
"Sentence_Break=CR": unicodeAliasSentence_Break_CR,
|
||||
@@ -90,6 +93,8 @@ var unicodeSupportedPropertyAliases = map[string]string{
|
||||
"extpict": "Extended_Pictographic",
|
||||
"gcb": "Grapheme_Cluster_Break",
|
||||
"graphemeclusterbreak": "Grapheme_Cluster_Break",
|
||||
"incb": "Indic_Conjunct_Break",
|
||||
"indicconjunctbreak": "Indic_Conjunct_Break",
|
||||
"math": "Math",
|
||||
"sb": "Sentence_Break",
|
||||
"sentencebreak": "Sentence_Break",
|
||||
@@ -118,6 +123,11 @@ var unicodeSupportedPropertyValueAliases = map[string]map[string]string{
|
||||
"v": "V",
|
||||
"zwj": "ZWJ",
|
||||
},
|
||||
"Indic_Conjunct_Break": {
|
||||
"consonant": "Consonant",
|
||||
"extend": "Extend",
|
||||
"linker": "Linker",
|
||||
},
|
||||
"Sentence_Break": {
|
||||
"at": "ATerm",
|
||||
"aterm": "ATerm",
|
||||
@@ -6056,3 +6066,529 @@ var unicodeAliasWord_Break_ZWJ = &unicode.RangeTable{
|
||||
{Lo: 0x200D, Hi: 0x200D, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Consonant = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x915, Hi: 0x939, Stride: 1},
|
||||
{Lo: 0x958, Hi: 0x95F, Stride: 1},
|
||||
{Lo: 0x978, Hi: 0x97F, Stride: 1},
|
||||
{Lo: 0x995, Hi: 0x9A8, Stride: 1},
|
||||
{Lo: 0x9AA, Hi: 0x9B0, Stride: 1},
|
||||
{Lo: 0x9B2, Hi: 0x9B2, Stride: 1},
|
||||
{Lo: 0x9B6, Hi: 0x9B9, Stride: 1},
|
||||
{Lo: 0x9DC, Hi: 0x9DD, Stride: 1},
|
||||
{Lo: 0x9DF, Hi: 0x9DF, Stride: 1},
|
||||
{Lo: 0x9F0, Hi: 0x9F1, Stride: 1},
|
||||
{Lo: 0xA95, Hi: 0xAA8, Stride: 1},
|
||||
{Lo: 0xAAA, Hi: 0xAB0, Stride: 1},
|
||||
{Lo: 0xAB2, Hi: 0xAB3, Stride: 1},
|
||||
{Lo: 0xAB5, Hi: 0xAB9, Stride: 1},
|
||||
{Lo: 0xAF9, Hi: 0xAF9, Stride: 1},
|
||||
{Lo: 0xB15, Hi: 0xB28, Stride: 1},
|
||||
{Lo: 0xB2A, Hi: 0xB30, Stride: 1},
|
||||
{Lo: 0xB32, Hi: 0xB33, Stride: 1},
|
||||
{Lo: 0xB35, Hi: 0xB39, Stride: 1},
|
||||
{Lo: 0xB5C, Hi: 0xB5D, Stride: 1},
|
||||
{Lo: 0xB5F, Hi: 0xB5F, Stride: 1},
|
||||
{Lo: 0xB71, Hi: 0xB71, Stride: 1},
|
||||
{Lo: 0xC15, Hi: 0xC28, Stride: 1},
|
||||
{Lo: 0xC2A, Hi: 0xC39, Stride: 1},
|
||||
{Lo: 0xC58, Hi: 0xC5A, Stride: 1},
|
||||
{Lo: 0xD15, Hi: 0xD3A, Stride: 1},
|
||||
{Lo: 0x1000, Hi: 0x102A, Stride: 1},
|
||||
{Lo: 0x103F, Hi: 0x103F, Stride: 1},
|
||||
{Lo: 0x1050, Hi: 0x1055, Stride: 1},
|
||||
{Lo: 0x105A, Hi: 0x105D, Stride: 1},
|
||||
{Lo: 0x1061, Hi: 0x1061, Stride: 1},
|
||||
{Lo: 0x1065, Hi: 0x1066, Stride: 1},
|
||||
{Lo: 0x106E, Hi: 0x1070, Stride: 1},
|
||||
{Lo: 0x1075, Hi: 0x1081, Stride: 1},
|
||||
{Lo: 0x108E, Hi: 0x108E, Stride: 1},
|
||||
{Lo: 0x1780, Hi: 0x17B3, Stride: 1},
|
||||
{Lo: 0x1A20, Hi: 0x1A54, Stride: 1},
|
||||
{Lo: 0x1B0B, Hi: 0x1B0C, Stride: 1},
|
||||
{Lo: 0x1B13, Hi: 0x1B33, Stride: 1},
|
||||
{Lo: 0x1B45, Hi: 0x1B4C, Stride: 1},
|
||||
{Lo: 0x1B83, Hi: 0x1BA0, Stride: 1},
|
||||
{Lo: 0x1BAE, Hi: 0x1BAF, Stride: 1},
|
||||
{Lo: 0x1BBB, Hi: 0x1BBD, Stride: 1},
|
||||
{Lo: 0xA989, Hi: 0xA98B, Stride: 1},
|
||||
{Lo: 0xA98F, Hi: 0xA9B2, Stride: 1},
|
||||
{Lo: 0xA9E0, Hi: 0xA9E4, Stride: 1},
|
||||
{Lo: 0xA9E7, Hi: 0xA9EF, Stride: 1},
|
||||
{Lo: 0xA9FA, Hi: 0xA9FE, Stride: 1},
|
||||
{Lo: 0xAA60, Hi: 0xAA6F, Stride: 1},
|
||||
{Lo: 0xAA71, Hi: 0xAA73, Stride: 1},
|
||||
{Lo: 0xAA7A, Hi: 0xAA7A, Stride: 1},
|
||||
{Lo: 0xAA7E, Hi: 0xAA7F, Stride: 1},
|
||||
{Lo: 0xAAE0, Hi: 0xAAEA, Stride: 1},
|
||||
{Lo: 0xABC0, Hi: 0xABDA, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x10A00, Hi: 0x10A00, Stride: 1},
|
||||
{Lo: 0x10A10, Hi: 0x10A13, Stride: 1},
|
||||
{Lo: 0x10A15, Hi: 0x10A17, Stride: 1},
|
||||
{Lo: 0x10A19, Hi: 0x10A35, Stride: 1},
|
||||
{Lo: 0x11103, Hi: 0x11126, Stride: 1},
|
||||
{Lo: 0x11144, Hi: 0x11144, Stride: 1},
|
||||
{Lo: 0x11147, Hi: 0x11147, Stride: 1},
|
||||
{Lo: 0x11380, Hi: 0x11389, Stride: 1},
|
||||
{Lo: 0x1138B, Hi: 0x1138B, Stride: 1},
|
||||
{Lo: 0x1138E, Hi: 0x1138E, Stride: 1},
|
||||
{Lo: 0x11390, Hi: 0x113B5, Stride: 1},
|
||||
{Lo: 0x11900, Hi: 0x11906, Stride: 1},
|
||||
{Lo: 0x11909, Hi: 0x11909, Stride: 1},
|
||||
{Lo: 0x1190C, Hi: 0x11913, Stride: 1},
|
||||
{Lo: 0x11915, Hi: 0x11916, Stride: 1},
|
||||
{Lo: 0x11918, Hi: 0x1192F, Stride: 1},
|
||||
{Lo: 0x11A00, Hi: 0x11A00, Stride: 1},
|
||||
{Lo: 0x11A0B, Hi: 0x11A32, Stride: 1},
|
||||
{Lo: 0x11A50, Hi: 0x11A50, Stride: 1},
|
||||
{Lo: 0x11A5C, Hi: 0x11A83, Stride: 1},
|
||||
{Lo: 0x11F04, Hi: 0x11F10, Stride: 1},
|
||||
{Lo: 0x11F12, Hi: 0x11F33, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Extend = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x300, Hi: 0x36F, Stride: 1},
|
||||
{Lo: 0x483, Hi: 0x487, Stride: 1},
|
||||
{Lo: 0x488, Hi: 0x489, Stride: 1},
|
||||
{Lo: 0x591, Hi: 0x5BD, Stride: 1},
|
||||
{Lo: 0x5BF, Hi: 0x5BF, Stride: 1},
|
||||
{Lo: 0x5C1, Hi: 0x5C2, Stride: 1},
|
||||
{Lo: 0x5C4, Hi: 0x5C5, Stride: 1},
|
||||
{Lo: 0x5C7, Hi: 0x5C7, Stride: 1},
|
||||
{Lo: 0x610, Hi: 0x61A, Stride: 1},
|
||||
{Lo: 0x64B, Hi: 0x65F, Stride: 1},
|
||||
{Lo: 0x670, Hi: 0x670, Stride: 1},
|
||||
{Lo: 0x6D6, Hi: 0x6DC, Stride: 1},
|
||||
{Lo: 0x6DF, Hi: 0x6E4, Stride: 1},
|
||||
{Lo: 0x6E7, Hi: 0x6E8, Stride: 1},
|
||||
{Lo: 0x6EA, Hi: 0x6ED, Stride: 1},
|
||||
{Lo: 0x711, Hi: 0x711, Stride: 1},
|
||||
{Lo: 0x730, Hi: 0x74A, Stride: 1},
|
||||
{Lo: 0x7A6, Hi: 0x7B0, Stride: 1},
|
||||
{Lo: 0x7EB, Hi: 0x7F3, Stride: 1},
|
||||
{Lo: 0x7FD, Hi: 0x7FD, Stride: 1},
|
||||
{Lo: 0x816, Hi: 0x819, Stride: 1},
|
||||
{Lo: 0x81B, Hi: 0x823, Stride: 1},
|
||||
{Lo: 0x825, Hi: 0x827, Stride: 1},
|
||||
{Lo: 0x829, Hi: 0x82D, Stride: 1},
|
||||
{Lo: 0x859, Hi: 0x85B, Stride: 1},
|
||||
{Lo: 0x897, Hi: 0x89F, Stride: 1},
|
||||
{Lo: 0x8CA, Hi: 0x8E1, Stride: 1},
|
||||
{Lo: 0x8E3, Hi: 0x902, Stride: 1},
|
||||
{Lo: 0x93A, Hi: 0x93A, Stride: 1},
|
||||
{Lo: 0x93C, Hi: 0x93C, Stride: 1},
|
||||
{Lo: 0x941, Hi: 0x948, Stride: 1},
|
||||
{Lo: 0x951, Hi: 0x957, Stride: 1},
|
||||
{Lo: 0x962, Hi: 0x963, Stride: 1},
|
||||
{Lo: 0x981, Hi: 0x981, Stride: 1},
|
||||
{Lo: 0x9BC, Hi: 0x9BC, Stride: 1},
|
||||
{Lo: 0x9BE, Hi: 0x9BE, Stride: 1},
|
||||
{Lo: 0x9C1, Hi: 0x9C4, Stride: 1},
|
||||
{Lo: 0x9D7, Hi: 0x9D7, Stride: 1},
|
||||
{Lo: 0x9E2, Hi: 0x9E3, Stride: 1},
|
||||
{Lo: 0x9FE, Hi: 0x9FE, Stride: 1},
|
||||
{Lo: 0xA01, Hi: 0xA02, Stride: 1},
|
||||
{Lo: 0xA3C, Hi: 0xA3C, Stride: 1},
|
||||
{Lo: 0xA41, Hi: 0xA42, Stride: 1},
|
||||
{Lo: 0xA47, Hi: 0xA48, Stride: 1},
|
||||
{Lo: 0xA4B, Hi: 0xA4D, Stride: 1},
|
||||
{Lo: 0xA51, Hi: 0xA51, Stride: 1},
|
||||
{Lo: 0xA70, Hi: 0xA71, Stride: 1},
|
||||
{Lo: 0xA75, Hi: 0xA75, Stride: 1},
|
||||
{Lo: 0xA81, Hi: 0xA82, Stride: 1},
|
||||
{Lo: 0xABC, Hi: 0xABC, Stride: 1},
|
||||
{Lo: 0xAC1, Hi: 0xAC5, Stride: 1},
|
||||
{Lo: 0xAC7, Hi: 0xAC8, Stride: 1},
|
||||
{Lo: 0xAE2, Hi: 0xAE3, Stride: 1},
|
||||
{Lo: 0xAFA, Hi: 0xAFF, Stride: 1},
|
||||
{Lo: 0xB01, Hi: 0xB01, Stride: 1},
|
||||
{Lo: 0xB3C, Hi: 0xB3C, Stride: 1},
|
||||
{Lo: 0xB3E, Hi: 0xB3E, Stride: 1},
|
||||
{Lo: 0xB3F, Hi: 0xB3F, Stride: 1},
|
||||
{Lo: 0xB41, Hi: 0xB44, Stride: 1},
|
||||
{Lo: 0xB55, Hi: 0xB56, Stride: 1},
|
||||
{Lo: 0xB57, Hi: 0xB57, Stride: 1},
|
||||
{Lo: 0xB62, Hi: 0xB63, Stride: 1},
|
||||
{Lo: 0xB82, Hi: 0xB82, Stride: 1},
|
||||
{Lo: 0xBBE, Hi: 0xBBE, Stride: 1},
|
||||
{Lo: 0xBC0, Hi: 0xBC0, Stride: 1},
|
||||
{Lo: 0xBCD, Hi: 0xBCD, Stride: 1},
|
||||
{Lo: 0xBD7, Hi: 0xBD7, Stride: 1},
|
||||
{Lo: 0xC00, Hi: 0xC00, Stride: 1},
|
||||
{Lo: 0xC04, Hi: 0xC04, Stride: 1},
|
||||
{Lo: 0xC3C, Hi: 0xC3C, Stride: 1},
|
||||
{Lo: 0xC3E, Hi: 0xC40, Stride: 1},
|
||||
{Lo: 0xC46, Hi: 0xC48, Stride: 1},
|
||||
{Lo: 0xC4A, Hi: 0xC4C, Stride: 1},
|
||||
{Lo: 0xC55, Hi: 0xC56, Stride: 1},
|
||||
{Lo: 0xC62, Hi: 0xC63, Stride: 1},
|
||||
{Lo: 0xC81, Hi: 0xC81, Stride: 1},
|
||||
{Lo: 0xCBC, Hi: 0xCBC, Stride: 1},
|
||||
{Lo: 0xCBF, Hi: 0xCBF, Stride: 1},
|
||||
{Lo: 0xCC0, Hi: 0xCC0, Stride: 1},
|
||||
{Lo: 0xCC2, Hi: 0xCC2, Stride: 1},
|
||||
{Lo: 0xCC6, Hi: 0xCC6, Stride: 1},
|
||||
{Lo: 0xCC7, Hi: 0xCC8, Stride: 1},
|
||||
{Lo: 0xCCA, Hi: 0xCCB, Stride: 1},
|
||||
{Lo: 0xCCC, Hi: 0xCCD, Stride: 1},
|
||||
{Lo: 0xCD5, Hi: 0xCD6, Stride: 1},
|
||||
{Lo: 0xCE2, Hi: 0xCE3, Stride: 1},
|
||||
{Lo: 0xD00, Hi: 0xD01, Stride: 1},
|
||||
{Lo: 0xD3B, Hi: 0xD3C, Stride: 1},
|
||||
{Lo: 0xD3E, Hi: 0xD3E, Stride: 1},
|
||||
{Lo: 0xD41, Hi: 0xD44, Stride: 1},
|
||||
{Lo: 0xD57, Hi: 0xD57, Stride: 1},
|
||||
{Lo: 0xD62, Hi: 0xD63, Stride: 1},
|
||||
{Lo: 0xD81, Hi: 0xD81, Stride: 1},
|
||||
{Lo: 0xDCA, Hi: 0xDCA, Stride: 1},
|
||||
{Lo: 0xDCF, Hi: 0xDCF, Stride: 1},
|
||||
{Lo: 0xDD2, Hi: 0xDD4, Stride: 1},
|
||||
{Lo: 0xDD6, Hi: 0xDD6, Stride: 1},
|
||||
{Lo: 0xDDF, Hi: 0xDDF, Stride: 1},
|
||||
{Lo: 0xE31, Hi: 0xE31, Stride: 1},
|
||||
{Lo: 0xE34, Hi: 0xE3A, Stride: 1},
|
||||
{Lo: 0xE47, Hi: 0xE4E, Stride: 1},
|
||||
{Lo: 0xEB1, Hi: 0xEB1, Stride: 1},
|
||||
{Lo: 0xEB4, Hi: 0xEBC, Stride: 1},
|
||||
{Lo: 0xEC8, Hi: 0xECE, Stride: 1},
|
||||
{Lo: 0xF18, Hi: 0xF19, Stride: 1},
|
||||
{Lo: 0xF35, Hi: 0xF35, Stride: 1},
|
||||
{Lo: 0xF37, Hi: 0xF37, Stride: 1},
|
||||
{Lo: 0xF39, Hi: 0xF39, Stride: 1},
|
||||
{Lo: 0xF71, Hi: 0xF7E, Stride: 1},
|
||||
{Lo: 0xF80, Hi: 0xF84, Stride: 1},
|
||||
{Lo: 0xF86, Hi: 0xF87, Stride: 1},
|
||||
{Lo: 0xF8D, Hi: 0xF97, Stride: 1},
|
||||
{Lo: 0xF99, Hi: 0xFBC, Stride: 1},
|
||||
{Lo: 0xFC6, Hi: 0xFC6, Stride: 1},
|
||||
{Lo: 0x102D, Hi: 0x1030, Stride: 1},
|
||||
{Lo: 0x1032, Hi: 0x1037, Stride: 1},
|
||||
{Lo: 0x103A, Hi: 0x103A, Stride: 1},
|
||||
{Lo: 0x103D, Hi: 0x103E, Stride: 1},
|
||||
{Lo: 0x1058, Hi: 0x1059, Stride: 1},
|
||||
{Lo: 0x105E, Hi: 0x1060, Stride: 1},
|
||||
{Lo: 0x1071, Hi: 0x1074, Stride: 1},
|
||||
{Lo: 0x1082, Hi: 0x1082, Stride: 1},
|
||||
{Lo: 0x1085, Hi: 0x1086, Stride: 1},
|
||||
{Lo: 0x108D, Hi: 0x108D, Stride: 1},
|
||||
{Lo: 0x109D, Hi: 0x109D, Stride: 1},
|
||||
{Lo: 0x135D, Hi: 0x135F, Stride: 1},
|
||||
{Lo: 0x1712, Hi: 0x1714, Stride: 1},
|
||||
{Lo: 0x1715, Hi: 0x1715, Stride: 1},
|
||||
{Lo: 0x1732, Hi: 0x1733, Stride: 1},
|
||||
{Lo: 0x1734, Hi: 0x1734, Stride: 1},
|
||||
{Lo: 0x1752, Hi: 0x1753, Stride: 1},
|
||||
{Lo: 0x1772, Hi: 0x1773, Stride: 1},
|
||||
{Lo: 0x17B4, Hi: 0x17B5, Stride: 1},
|
||||
{Lo: 0x17B7, Hi: 0x17BD, Stride: 1},
|
||||
{Lo: 0x17C6, Hi: 0x17C6, Stride: 1},
|
||||
{Lo: 0x17C9, Hi: 0x17D1, Stride: 1},
|
||||
{Lo: 0x17D3, Hi: 0x17D3, Stride: 1},
|
||||
{Lo: 0x17DD, Hi: 0x17DD, Stride: 1},
|
||||
{Lo: 0x180B, Hi: 0x180D, Stride: 1},
|
||||
{Lo: 0x180F, Hi: 0x180F, Stride: 1},
|
||||
{Lo: 0x1885, Hi: 0x1886, Stride: 1},
|
||||
{Lo: 0x18A9, Hi: 0x18A9, Stride: 1},
|
||||
{Lo: 0x1920, Hi: 0x1922, Stride: 1},
|
||||
{Lo: 0x1927, Hi: 0x1928, Stride: 1},
|
||||
{Lo: 0x1932, Hi: 0x1932, Stride: 1},
|
||||
{Lo: 0x1939, Hi: 0x193B, Stride: 1},
|
||||
{Lo: 0x1A17, Hi: 0x1A18, Stride: 1},
|
||||
{Lo: 0x1A1B, Hi: 0x1A1B, Stride: 1},
|
||||
{Lo: 0x1A56, Hi: 0x1A56, Stride: 1},
|
||||
{Lo: 0x1A58, Hi: 0x1A5E, Stride: 1},
|
||||
{Lo: 0x1A62, Hi: 0x1A62, Stride: 1},
|
||||
{Lo: 0x1A65, Hi: 0x1A6C, Stride: 1},
|
||||
{Lo: 0x1A73, Hi: 0x1A7C, Stride: 1},
|
||||
{Lo: 0x1A7F, Hi: 0x1A7F, Stride: 1},
|
||||
{Lo: 0x1AB0, Hi: 0x1ABD, Stride: 1},
|
||||
{Lo: 0x1ABE, Hi: 0x1ABE, Stride: 1},
|
||||
{Lo: 0x1ABF, Hi: 0x1ADD, Stride: 1},
|
||||
{Lo: 0x1AE0, Hi: 0x1AEB, Stride: 1},
|
||||
{Lo: 0x1B00, Hi: 0x1B03, Stride: 1},
|
||||
{Lo: 0x1B34, Hi: 0x1B34, Stride: 1},
|
||||
{Lo: 0x1B35, Hi: 0x1B35, Stride: 1},
|
||||
{Lo: 0x1B36, Hi: 0x1B3A, Stride: 1},
|
||||
{Lo: 0x1B3B, Hi: 0x1B3B, Stride: 1},
|
||||
{Lo: 0x1B3C, Hi: 0x1B3C, Stride: 1},
|
||||
{Lo: 0x1B3D, Hi: 0x1B3D, Stride: 1},
|
||||
{Lo: 0x1B42, Hi: 0x1B42, Stride: 1},
|
||||
{Lo: 0x1B43, Hi: 0x1B43, Stride: 1},
|
||||
{Lo: 0x1B6B, Hi: 0x1B73, Stride: 1},
|
||||
{Lo: 0x1B80, Hi: 0x1B81, Stride: 1},
|
||||
{Lo: 0x1BA2, Hi: 0x1BA5, Stride: 1},
|
||||
{Lo: 0x1BA8, Hi: 0x1BA9, Stride: 1},
|
||||
{Lo: 0x1BAA, Hi: 0x1BAA, Stride: 1},
|
||||
{Lo: 0x1BAC, Hi: 0x1BAD, Stride: 1},
|
||||
{Lo: 0x1BE6, Hi: 0x1BE6, Stride: 1},
|
||||
{Lo: 0x1BE8, Hi: 0x1BE9, Stride: 1},
|
||||
{Lo: 0x1BED, Hi: 0x1BED, Stride: 1},
|
||||
{Lo: 0x1BEF, Hi: 0x1BF1, Stride: 1},
|
||||
{Lo: 0x1BF2, Hi: 0x1BF3, Stride: 1},
|
||||
{Lo: 0x1C2C, Hi: 0x1C33, Stride: 1},
|
||||
{Lo: 0x1C36, Hi: 0x1C37, Stride: 1},
|
||||
{Lo: 0x1CD0, Hi: 0x1CD2, Stride: 1},
|
||||
{Lo: 0x1CD4, Hi: 0x1CE0, Stride: 1},
|
||||
{Lo: 0x1CE2, Hi: 0x1CE8, Stride: 1},
|
||||
{Lo: 0x1CED, Hi: 0x1CED, Stride: 1},
|
||||
{Lo: 0x1CF4, Hi: 0x1CF4, Stride: 1},
|
||||
{Lo: 0x1CF8, Hi: 0x1CF9, Stride: 1},
|
||||
{Lo: 0x1DC0, Hi: 0x1DFF, Stride: 1},
|
||||
{Lo: 0x200D, Hi: 0x200D, Stride: 1},
|
||||
{Lo: 0x20D0, Hi: 0x20DC, Stride: 1},
|
||||
{Lo: 0x20DD, Hi: 0x20E0, Stride: 1},
|
||||
{Lo: 0x20E1, Hi: 0x20E1, Stride: 1},
|
||||
{Lo: 0x20E2, Hi: 0x20E4, Stride: 1},
|
||||
{Lo: 0x20E5, Hi: 0x20F0, Stride: 1},
|
||||
{Lo: 0x2CEF, Hi: 0x2CF1, Stride: 1},
|
||||
{Lo: 0x2D7F, Hi: 0x2D7F, Stride: 1},
|
||||
{Lo: 0x2DE0, Hi: 0x2DFF, Stride: 1},
|
||||
{Lo: 0x302A, Hi: 0x302D, Stride: 1},
|
||||
{Lo: 0x302E, Hi: 0x302F, Stride: 1},
|
||||
{Lo: 0x3099, Hi: 0x309A, Stride: 1},
|
||||
{Lo: 0xA66F, Hi: 0xA66F, Stride: 1},
|
||||
{Lo: 0xA670, Hi: 0xA672, Stride: 1},
|
||||
{Lo: 0xA674, Hi: 0xA67D, Stride: 1},
|
||||
{Lo: 0xA69E, Hi: 0xA69F, Stride: 1},
|
||||
{Lo: 0xA6F0, Hi: 0xA6F1, Stride: 1},
|
||||
{Lo: 0xA802, Hi: 0xA802, Stride: 1},
|
||||
{Lo: 0xA806, Hi: 0xA806, Stride: 1},
|
||||
{Lo: 0xA80B, Hi: 0xA80B, Stride: 1},
|
||||
{Lo: 0xA825, Hi: 0xA826, Stride: 1},
|
||||
{Lo: 0xA82C, Hi: 0xA82C, Stride: 1},
|
||||
{Lo: 0xA8C4, Hi: 0xA8C5, Stride: 1},
|
||||
{Lo: 0xA8E0, Hi: 0xA8F1, Stride: 1},
|
||||
{Lo: 0xA8FF, Hi: 0xA8FF, Stride: 1},
|
||||
{Lo: 0xA926, Hi: 0xA92D, Stride: 1},
|
||||
{Lo: 0xA947, Hi: 0xA951, Stride: 1},
|
||||
{Lo: 0xA953, Hi: 0xA953, Stride: 1},
|
||||
{Lo: 0xA980, Hi: 0xA982, Stride: 1},
|
||||
{Lo: 0xA9B3, Hi: 0xA9B3, Stride: 1},
|
||||
{Lo: 0xA9B6, Hi: 0xA9B9, Stride: 1},
|
||||
{Lo: 0xA9BC, Hi: 0xA9BD, Stride: 1},
|
||||
{Lo: 0xA9E5, Hi: 0xA9E5, Stride: 1},
|
||||
{Lo: 0xAA29, Hi: 0xAA2E, Stride: 1},
|
||||
{Lo: 0xAA31, Hi: 0xAA32, Stride: 1},
|
||||
{Lo: 0xAA35, Hi: 0xAA36, Stride: 1},
|
||||
{Lo: 0xAA43, Hi: 0xAA43, Stride: 1},
|
||||
{Lo: 0xAA4C, Hi: 0xAA4C, Stride: 1},
|
||||
{Lo: 0xAA7C, Hi: 0xAA7C, Stride: 1},
|
||||
{Lo: 0xAAB0, Hi: 0xAAB0, Stride: 1},
|
||||
{Lo: 0xAAB2, Hi: 0xAAB4, Stride: 1},
|
||||
{Lo: 0xAAB7, Hi: 0xAAB8, Stride: 1},
|
||||
{Lo: 0xAABE, Hi: 0xAABF, Stride: 1},
|
||||
{Lo: 0xAAC1, Hi: 0xAAC1, Stride: 1},
|
||||
{Lo: 0xAAEC, Hi: 0xAAED, Stride: 1},
|
||||
{Lo: 0xABE5, Hi: 0xABE5, Stride: 1},
|
||||
{Lo: 0xABE8, Hi: 0xABE8, Stride: 1},
|
||||
{Lo: 0xABED, Hi: 0xABED, Stride: 1},
|
||||
{Lo: 0xFB1E, Hi: 0xFB1E, Stride: 1},
|
||||
{Lo: 0xFE00, Hi: 0xFE0F, Stride: 1},
|
||||
{Lo: 0xFE20, Hi: 0xFE2F, Stride: 1},
|
||||
{Lo: 0xFF9E, Hi: 0xFF9F, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x101FD, Hi: 0x101FD, Stride: 1},
|
||||
{Lo: 0x102E0, Hi: 0x102E0, Stride: 1},
|
||||
{Lo: 0x10376, Hi: 0x1037A, Stride: 1},
|
||||
{Lo: 0x10A01, Hi: 0x10A03, Stride: 1},
|
||||
{Lo: 0x10A05, Hi: 0x10A06, Stride: 1},
|
||||
{Lo: 0x10A0C, Hi: 0x10A0F, Stride: 1},
|
||||
{Lo: 0x10A38, Hi: 0x10A3A, Stride: 1},
|
||||
{Lo: 0x10AE5, Hi: 0x10AE6, Stride: 1},
|
||||
{Lo: 0x10D24, Hi: 0x10D27, Stride: 1},
|
||||
{Lo: 0x10D69, Hi: 0x10D6D, Stride: 1},
|
||||
{Lo: 0x10EAB, Hi: 0x10EAC, Stride: 1},
|
||||
{Lo: 0x10EFA, Hi: 0x10EFF, Stride: 1},
|
||||
{Lo: 0x10F46, Hi: 0x10F50, Stride: 1},
|
||||
{Lo: 0x10F82, Hi: 0x10F85, Stride: 1},
|
||||
{Lo: 0x11001, Hi: 0x11001, Stride: 1},
|
||||
{Lo: 0x11038, Hi: 0x11046, Stride: 1},
|
||||
{Lo: 0x11070, Hi: 0x11070, Stride: 1},
|
||||
{Lo: 0x11073, Hi: 0x11074, Stride: 1},
|
||||
{Lo: 0x1107F, Hi: 0x11081, Stride: 1},
|
||||
{Lo: 0x110B3, Hi: 0x110B6, Stride: 1},
|
||||
{Lo: 0x110B9, Hi: 0x110BA, Stride: 1},
|
||||
{Lo: 0x110C2, Hi: 0x110C2, Stride: 1},
|
||||
{Lo: 0x11100, Hi: 0x11102, Stride: 1},
|
||||
{Lo: 0x11127, Hi: 0x1112B, Stride: 1},
|
||||
{Lo: 0x1112D, Hi: 0x11132, Stride: 1},
|
||||
{Lo: 0x11134, Hi: 0x11134, Stride: 1},
|
||||
{Lo: 0x11173, Hi: 0x11173, Stride: 1},
|
||||
{Lo: 0x11180, Hi: 0x11181, Stride: 1},
|
||||
{Lo: 0x111B6, Hi: 0x111BE, Stride: 1},
|
||||
{Lo: 0x111C0, Hi: 0x111C0, Stride: 1},
|
||||
{Lo: 0x111C9, Hi: 0x111CC, Stride: 1},
|
||||
{Lo: 0x111CF, Hi: 0x111CF, Stride: 1},
|
||||
{Lo: 0x1122F, Hi: 0x11231, Stride: 1},
|
||||
{Lo: 0x11234, Hi: 0x11234, Stride: 1},
|
||||
{Lo: 0x11235, Hi: 0x11235, Stride: 1},
|
||||
{Lo: 0x11236, Hi: 0x11237, Stride: 1},
|
||||
{Lo: 0x1123E, Hi: 0x1123E, Stride: 1},
|
||||
{Lo: 0x11241, Hi: 0x11241, Stride: 1},
|
||||
{Lo: 0x112DF, Hi: 0x112DF, Stride: 1},
|
||||
{Lo: 0x112E3, Hi: 0x112EA, Stride: 1},
|
||||
{Lo: 0x11300, Hi: 0x11301, Stride: 1},
|
||||
{Lo: 0x1133B, Hi: 0x1133C, Stride: 1},
|
||||
{Lo: 0x1133E, Hi: 0x1133E, Stride: 1},
|
||||
{Lo: 0x11340, Hi: 0x11340, Stride: 1},
|
||||
{Lo: 0x1134D, Hi: 0x1134D, Stride: 1},
|
||||
{Lo: 0x11357, Hi: 0x11357, Stride: 1},
|
||||
{Lo: 0x11366, Hi: 0x1136C, Stride: 1},
|
||||
{Lo: 0x11370, Hi: 0x11374, Stride: 1},
|
||||
{Lo: 0x113B8, Hi: 0x113B8, Stride: 1},
|
||||
{Lo: 0x113BB, Hi: 0x113C0, Stride: 1},
|
||||
{Lo: 0x113C2, Hi: 0x113C2, Stride: 1},
|
||||
{Lo: 0x113C5, Hi: 0x113C5, Stride: 1},
|
||||
{Lo: 0x113C7, Hi: 0x113C9, Stride: 1},
|
||||
{Lo: 0x113CE, Hi: 0x113CE, Stride: 1},
|
||||
{Lo: 0x113CF, Hi: 0x113CF, Stride: 1},
|
||||
{Lo: 0x113D2, Hi: 0x113D2, Stride: 1},
|
||||
{Lo: 0x113E1, Hi: 0x113E2, Stride: 1},
|
||||
{Lo: 0x11438, Hi: 0x1143F, Stride: 1},
|
||||
{Lo: 0x11442, Hi: 0x11444, Stride: 1},
|
||||
{Lo: 0x11446, Hi: 0x11446, Stride: 1},
|
||||
{Lo: 0x1145E, Hi: 0x1145E, Stride: 1},
|
||||
{Lo: 0x114B0, Hi: 0x114B0, Stride: 1},
|
||||
{Lo: 0x114B3, Hi: 0x114B8, Stride: 1},
|
||||
{Lo: 0x114BA, Hi: 0x114BA, Stride: 1},
|
||||
{Lo: 0x114BD, Hi: 0x114BD, Stride: 1},
|
||||
{Lo: 0x114BF, Hi: 0x114C0, Stride: 1},
|
||||
{Lo: 0x114C2, Hi: 0x114C3, Stride: 1},
|
||||
{Lo: 0x115AF, Hi: 0x115AF, Stride: 1},
|
||||
{Lo: 0x115B2, Hi: 0x115B5, Stride: 1},
|
||||
{Lo: 0x115BC, Hi: 0x115BD, Stride: 1},
|
||||
{Lo: 0x115BF, Hi: 0x115C0, Stride: 1},
|
||||
{Lo: 0x115DC, Hi: 0x115DD, Stride: 1},
|
||||
{Lo: 0x11633, Hi: 0x1163A, Stride: 1},
|
||||
{Lo: 0x1163D, Hi: 0x1163D, Stride: 1},
|
||||
{Lo: 0x1163F, Hi: 0x11640, Stride: 1},
|
||||
{Lo: 0x116AB, Hi: 0x116AB, Stride: 1},
|
||||
{Lo: 0x116AD, Hi: 0x116AD, Stride: 1},
|
||||
{Lo: 0x116B0, Hi: 0x116B5, Stride: 1},
|
||||
{Lo: 0x116B6, Hi: 0x116B6, Stride: 1},
|
||||
{Lo: 0x116B7, Hi: 0x116B7, Stride: 1},
|
||||
{Lo: 0x1171D, Hi: 0x1171D, Stride: 1},
|
||||
{Lo: 0x1171F, Hi: 0x1171F, Stride: 1},
|
||||
{Lo: 0x11722, Hi: 0x11725, Stride: 1},
|
||||
{Lo: 0x11727, Hi: 0x1172B, Stride: 1},
|
||||
{Lo: 0x1182F, Hi: 0x11837, Stride: 1},
|
||||
{Lo: 0x11839, Hi: 0x1183A, Stride: 1},
|
||||
{Lo: 0x11930, Hi: 0x11930, Stride: 1},
|
||||
{Lo: 0x1193B, Hi: 0x1193C, Stride: 1},
|
||||
{Lo: 0x1193D, Hi: 0x1193D, Stride: 1},
|
||||
{Lo: 0x11943, Hi: 0x11943, Stride: 1},
|
||||
{Lo: 0x119D4, Hi: 0x119D7, Stride: 1},
|
||||
{Lo: 0x119DA, Hi: 0x119DB, Stride: 1},
|
||||
{Lo: 0x119E0, Hi: 0x119E0, Stride: 1},
|
||||
{Lo: 0x11A01, Hi: 0x11A0A, Stride: 1},
|
||||
{Lo: 0x11A33, Hi: 0x11A38, Stride: 1},
|
||||
{Lo: 0x11A3B, Hi: 0x11A3E, Stride: 1},
|
||||
{Lo: 0x11A51, Hi: 0x11A56, Stride: 1},
|
||||
{Lo: 0x11A59, Hi: 0x11A5B, Stride: 1},
|
||||
{Lo: 0x11A8A, Hi: 0x11A96, Stride: 1},
|
||||
{Lo: 0x11A98, Hi: 0x11A98, Stride: 1},
|
||||
{Lo: 0x11B60, Hi: 0x11B60, Stride: 1},
|
||||
{Lo: 0x11B62, Hi: 0x11B64, Stride: 1},
|
||||
{Lo: 0x11B66, Hi: 0x11B66, Stride: 1},
|
||||
{Lo: 0x11C30, Hi: 0x11C36, Stride: 1},
|
||||
{Lo: 0x11C38, Hi: 0x11C3D, Stride: 1},
|
||||
{Lo: 0x11C3F, Hi: 0x11C3F, Stride: 1},
|
||||
{Lo: 0x11C92, Hi: 0x11CA7, Stride: 1},
|
||||
{Lo: 0x11CAA, Hi: 0x11CB0, Stride: 1},
|
||||
{Lo: 0x11CB2, Hi: 0x11CB3, Stride: 1},
|
||||
{Lo: 0x11CB5, Hi: 0x11CB6, Stride: 1},
|
||||
{Lo: 0x11D31, Hi: 0x11D36, Stride: 1},
|
||||
{Lo: 0x11D3A, Hi: 0x11D3A, Stride: 1},
|
||||
{Lo: 0x11D3C, Hi: 0x11D3D, Stride: 1},
|
||||
{Lo: 0x11D3F, Hi: 0x11D45, Stride: 1},
|
||||
{Lo: 0x11D47, Hi: 0x11D47, Stride: 1},
|
||||
{Lo: 0x11D90, Hi: 0x11D91, Stride: 1},
|
||||
{Lo: 0x11D95, Hi: 0x11D95, Stride: 1},
|
||||
{Lo: 0x11D97, Hi: 0x11D97, Stride: 1},
|
||||
{Lo: 0x11EF3, Hi: 0x11EF4, Stride: 1},
|
||||
{Lo: 0x11F00, Hi: 0x11F01, Stride: 1},
|
||||
{Lo: 0x11F36, Hi: 0x11F3A, Stride: 1},
|
||||
{Lo: 0x11F40, Hi: 0x11F40, Stride: 1},
|
||||
{Lo: 0x11F41, Hi: 0x11F41, Stride: 1},
|
||||
{Lo: 0x11F5A, Hi: 0x11F5A, Stride: 1},
|
||||
{Lo: 0x13440, Hi: 0x13440, Stride: 1},
|
||||
{Lo: 0x13447, Hi: 0x13455, Stride: 1},
|
||||
{Lo: 0x1611E, Hi: 0x16129, Stride: 1},
|
||||
{Lo: 0x1612D, Hi: 0x1612F, Stride: 1},
|
||||
{Lo: 0x16AF0, Hi: 0x16AF4, Stride: 1},
|
||||
{Lo: 0x16B30, Hi: 0x16B36, Stride: 1},
|
||||
{Lo: 0x16F4F, Hi: 0x16F4F, Stride: 1},
|
||||
{Lo: 0x16F8F, Hi: 0x16F92, Stride: 1},
|
||||
{Lo: 0x16FE4, Hi: 0x16FE4, Stride: 1},
|
||||
{Lo: 0x16FF0, Hi: 0x16FF1, Stride: 1},
|
||||
{Lo: 0x1BC9D, Hi: 0x1BC9E, Stride: 1},
|
||||
{Lo: 0x1CF00, Hi: 0x1CF2D, Stride: 1},
|
||||
{Lo: 0x1CF30, Hi: 0x1CF46, Stride: 1},
|
||||
{Lo: 0x1D165, Hi: 0x1D166, Stride: 1},
|
||||
{Lo: 0x1D167, Hi: 0x1D169, Stride: 1},
|
||||
{Lo: 0x1D16D, Hi: 0x1D172, Stride: 1},
|
||||
{Lo: 0x1D17B, Hi: 0x1D182, Stride: 1},
|
||||
{Lo: 0x1D185, Hi: 0x1D18B, Stride: 1},
|
||||
{Lo: 0x1D1AA, Hi: 0x1D1AD, Stride: 1},
|
||||
{Lo: 0x1D242, Hi: 0x1D244, Stride: 1},
|
||||
{Lo: 0x1DA00, Hi: 0x1DA36, Stride: 1},
|
||||
{Lo: 0x1DA3B, Hi: 0x1DA6C, Stride: 1},
|
||||
{Lo: 0x1DA75, Hi: 0x1DA75, Stride: 1},
|
||||
{Lo: 0x1DA84, Hi: 0x1DA84, Stride: 1},
|
||||
{Lo: 0x1DA9B, Hi: 0x1DA9F, Stride: 1},
|
||||
{Lo: 0x1DAA1, Hi: 0x1DAAF, Stride: 1},
|
||||
{Lo: 0x1E000, Hi: 0x1E006, Stride: 1},
|
||||
{Lo: 0x1E008, Hi: 0x1E018, Stride: 1},
|
||||
{Lo: 0x1E01B, Hi: 0x1E021, Stride: 1},
|
||||
{Lo: 0x1E023, Hi: 0x1E024, Stride: 1},
|
||||
{Lo: 0x1E026, Hi: 0x1E02A, Stride: 1},
|
||||
{Lo: 0x1E08F, Hi: 0x1E08F, Stride: 1},
|
||||
{Lo: 0x1E130, Hi: 0x1E136, Stride: 1},
|
||||
{Lo: 0x1E2AE, Hi: 0x1E2AE, Stride: 1},
|
||||
{Lo: 0x1E2EC, Hi: 0x1E2EF, Stride: 1},
|
||||
{Lo: 0x1E4EC, Hi: 0x1E4EF, Stride: 1},
|
||||
{Lo: 0x1E5EE, Hi: 0x1E5EF, Stride: 1},
|
||||
{Lo: 0x1E6E3, Hi: 0x1E6E3, Stride: 1},
|
||||
{Lo: 0x1E6E6, Hi: 0x1E6E6, Stride: 1},
|
||||
{Lo: 0x1E6EE, Hi: 0x1E6EF, Stride: 1},
|
||||
{Lo: 0x1E6F5, Hi: 0x1E6F5, Stride: 1},
|
||||
{Lo: 0x1E8D0, Hi: 0x1E8D6, Stride: 1},
|
||||
{Lo: 0x1E944, Hi: 0x1E94A, Stride: 1},
|
||||
{Lo: 0x1F3FB, Hi: 0x1F3FF, Stride: 1},
|
||||
{Lo: 0xE0020, Hi: 0xE007F, Stride: 1},
|
||||
{Lo: 0xE0100, Hi: 0xE01EF, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Linker = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x94D, Hi: 0x94D, Stride: 1},
|
||||
{Lo: 0x9CD, Hi: 0x9CD, Stride: 1},
|
||||
{Lo: 0xACD, Hi: 0xACD, Stride: 1},
|
||||
{Lo: 0xB4D, Hi: 0xB4D, Stride: 1},
|
||||
{Lo: 0xC4D, Hi: 0xC4D, Stride: 1},
|
||||
{Lo: 0xD4D, Hi: 0xD4D, Stride: 1},
|
||||
{Lo: 0x1039, Hi: 0x1039, Stride: 1},
|
||||
{Lo: 0x17D2, Hi: 0x17D2, Stride: 1},
|
||||
{Lo: 0x1A60, Hi: 0x1A60, Stride: 1},
|
||||
{Lo: 0x1B44, Hi: 0x1B44, Stride: 1},
|
||||
{Lo: 0x1BAB, Hi: 0x1BAB, Stride: 1},
|
||||
{Lo: 0xA9C0, Hi: 0xA9C0, Stride: 1},
|
||||
{Lo: 0xAAF6, Hi: 0xAAF6, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x10A3F, Hi: 0x10A3F, Stride: 1},
|
||||
{Lo: 0x11133, Hi: 0x11133, Stride: 1},
|
||||
{Lo: 0x113D0, Hi: 0x113D0, Stride: 1},
|
||||
{Lo: 0x1193E, Hi: 0x1193E, Stride: 1},
|
||||
{Lo: 0x11A47, Hi: 0x11A47, Stride: 1},
|
||||
{Lo: 0x11A99, Hi: 0x11A99, Stride: 1},
|
||||
{Lo: 0x11F42, Hi: 0x11F42, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
+231
-24
@@ -4,34 +4,93 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
)
|
||||
|
||||
func Write(tree *RegexTree) (*Code, error) {
|
||||
w := writer{
|
||||
intStack: make([]int, 0, 32),
|
||||
emitted: make([]int, 2),
|
||||
stringhash: make(map[string]int),
|
||||
sethash: make(map[string]int),
|
||||
w := newWriter(nil)
|
||||
code, err := w.codeFromTree(tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
code, err := w.codeFromTree(tree)
|
||||
if slices.Contains(code.CaptureSlotInUse, false) {
|
||||
quickWriter := newWriter(code.CaptureSlotInUse)
|
||||
quickCode, err := quickWriter.codeFromTree(tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !dispatchTablesEqual(code.Dispatches, quickCode.Dispatches) {
|
||||
return nil, fmt.Errorf("full and quick dispatch tables differ")
|
||||
}
|
||||
for i := range quickCode.Dispatches {
|
||||
quickCode.Dispatches[i].Sets = code.Dispatches[i].Sets
|
||||
quickCode.Dispatches[i].ASCII = code.Dispatches[i].ASCII
|
||||
quickCode.Dispatches[i].ASCIIMasks = code.Dispatches[i].ASCIIMasks
|
||||
}
|
||||
code.QuickCodes = quickCode.Codes
|
||||
code.QuickDispatches = quickCode.Dispatches
|
||||
}
|
||||
return code, nil
|
||||
}
|
||||
|
||||
return code, err
|
||||
func newWriter(quickCaptureSlots []bool) writer {
|
||||
return writer{
|
||||
intStack: make([]int, 0, 32),
|
||||
emitted: make([]int, 2),
|
||||
stringhash: make(map[string]int),
|
||||
sethash: make(map[string]int),
|
||||
preconsumed: make(map[*RegexNode]bool),
|
||||
dispatchInfo: make(map[*RegexNode]dispatchInfo),
|
||||
dispatchCodes: make(map[*RegexNode]int),
|
||||
dispatchGotos: make(map[*RegexNode][]int),
|
||||
quickCaptureSlots: quickCaptureSlots,
|
||||
}
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
emitted []int
|
||||
|
||||
intStack []int
|
||||
curpos int
|
||||
stringhash map[string]int
|
||||
stringtable [][]rune
|
||||
sethash map[string]int
|
||||
settable []*CharSet
|
||||
counting bool
|
||||
count int
|
||||
trackcount int
|
||||
caps map[int]int
|
||||
intStack []int
|
||||
curpos int
|
||||
stringhash map[string]int
|
||||
stringtable [][]rune
|
||||
sethash map[string]int
|
||||
settable []*CharSet
|
||||
dispatchtable []DispatchTable
|
||||
preconsumed map[*RegexNode]bool
|
||||
dispatchInfo map[*RegexNode]dispatchInfo
|
||||
dispatchCodes map[*RegexNode]int
|
||||
dispatchGotos map[*RegexNode][]int
|
||||
counting bool
|
||||
count int
|
||||
trackcount int
|
||||
caps map[int]int
|
||||
quickCaptureSlots []bool
|
||||
}
|
||||
|
||||
type dispatchInfo struct {
|
||||
sets []*CharSet
|
||||
leaders []*RegexNode
|
||||
complete []bool
|
||||
ok bool
|
||||
}
|
||||
|
||||
func dispatchTablesEqual(left, right []DispatchTable) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for i := range left {
|
||||
if !slices.Equal(left[i].Sets, right[i].Sets) ||
|
||||
!slices.Equal(left[i].ASCIIMasks, right[i].ASCIIMasks) ||
|
||||
(left[i].ASCII == nil) != (right[i].ASCII == nil) {
|
||||
return false
|
||||
}
|
||||
if left[i].ASCII != nil && *left[i].ASCII != *right[i].ASCII {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -74,6 +133,8 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
for {
|
||||
if !w.counting {
|
||||
w.emitted = make([]int, w.count)
|
||||
w.dispatchCodes = make(map[*RegexNode]int)
|
||||
w.dispatchGotos = make(map[*RegexNode][]int)
|
||||
}
|
||||
|
||||
curNode = tree.Root
|
||||
@@ -120,6 +181,9 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
|
||||
w.counting = false
|
||||
}
|
||||
if w.quickCaptureSlots != nil {
|
||||
return &Code{Codes: w.emitted, Dispatches: w.dispatchtable, TrackCount: w.trackcount}, nil
|
||||
}
|
||||
|
||||
fcPrefix := getFirstCharsPrefix(tree)
|
||||
prefix := getPrefix(tree)
|
||||
@@ -141,17 +205,57 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
Codes: w.emitted,
|
||||
Strings: w.stringtable,
|
||||
Sets: w.settable,
|
||||
Dispatches: w.dispatchtable,
|
||||
TrackCount: w.trackcount,
|
||||
Caps: w.caps,
|
||||
Capsize: capsize,
|
||||
CaptureSlotInUse: captureSlotsInUse(w.emitted, capsize),
|
||||
FcPrefix: fcPrefix,
|
||||
BmPrefix: bmPrefix,
|
||||
Anchors: getAnchors(tree),
|
||||
RightToLeft: rtl,
|
||||
FindOptimizations: tree.FindOptimizations,
|
||||
LeftContextRunes: AnalyzeLeftContext(tree.Root),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AnalyzeLeftContext returns how many runes before a candidate start the matcher
|
||||
// may inspect. Slicing the input down to that candidate is legal only when no
|
||||
// opcode depends on the original search origin or unbounded left text.
|
||||
//
|
||||
// 0 never looks left of the start position
|
||||
// 1 one previous rune is enough for \b, or to keep ^/\A from seeing
|
||||
// the candidate as the original start (they test leftchars()==0)
|
||||
// -1 do not slice: lookbehind, or \G (NtStart), which keys off textstart
|
||||
func AnalyzeLeftContext(n *RegexNode) int {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
need := 0
|
||||
switch n.T {
|
||||
case NtPosLook, NtNegLook:
|
||||
if n.Options&RightToLeft != 0 {
|
||||
return -1
|
||||
}
|
||||
case NtStart:
|
||||
return -1
|
||||
case NtBol, NtBeginning, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary:
|
||||
need = 1
|
||||
}
|
||||
|
||||
for _, child := range n.Children {
|
||||
childNeed := AnalyzeLeftContext(child)
|
||||
if childNeed < 0 {
|
||||
return -1
|
||||
}
|
||||
if childNeed > need {
|
||||
need = childNeed
|
||||
}
|
||||
}
|
||||
return need
|
||||
}
|
||||
|
||||
// The main RegexCode generator. It does a depth-first walk
|
||||
// through the tree and calls EmitFragment to emits code before
|
||||
// and after each child of an interior node, and at each leaf.
|
||||
@@ -171,13 +275,37 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtConcatenate | BeforeChild, NtConcatenate | AfterChild, NtEmpty:
|
||||
|
||||
case NtAlternate | BeforeChild:
|
||||
if curIndex < len(node.Children)-1 {
|
||||
if sets, leaders, _, ok := w.getDispatchCandidates(node); ok {
|
||||
if curIndex == 0 {
|
||||
w.dispatchCodes[node] = w.dispatchCode(sets)
|
||||
w.emit1(Dispatch|bits, w.dispatchCodes[node])
|
||||
}
|
||||
w.preconsumed[leaders[curIndex]] = true
|
||||
w.patchDispatchBranch(node, curIndex, w.curPos())
|
||||
} else if curIndex < len(node.Children)-1 {
|
||||
w.pushInt(w.curPos())
|
||||
w.emit1(Lazybranch, 0)
|
||||
}
|
||||
|
||||
case NtAlternate | AfterChild:
|
||||
if curIndex < len(node.Children)-1 {
|
||||
if _, _, complete, ok := w.getDispatchCandidates(node); ok {
|
||||
if curIndex < len(node.Children)-1 && !complete[curIndex] {
|
||||
gotoPos := w.curPos()
|
||||
w.emit1(Goto, 0)
|
||||
w.dispatchGotos[node] = append(w.dispatchGotos[node], gotoPos)
|
||||
}
|
||||
if curIndex == len(node.Children)-1 {
|
||||
end := w.curPos()
|
||||
for _, gotoPos := range w.dispatchGotos[node] {
|
||||
w.patchJump(gotoPos, end)
|
||||
}
|
||||
for branch, isComplete := range complete {
|
||||
if isComplete {
|
||||
w.patchDispatchBranch(node, branch, end)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if curIndex < len(node.Children)-1 {
|
||||
lbPos := w.popInt()
|
||||
w.pushInt(w.curPos())
|
||||
w.emit1(Goto, 0)
|
||||
@@ -281,10 +409,14 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtGroup | BeforeChild, NtGroup | AfterChild:
|
||||
|
||||
case NtCapture | BeforeChild:
|
||||
w.emit(Setmark)
|
||||
if w.emitCapture(node) {
|
||||
w.emit(Setmark)
|
||||
}
|
||||
|
||||
case NtCapture | AfterChild:
|
||||
w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N))
|
||||
if w.emitCapture(node) {
|
||||
w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N))
|
||||
}
|
||||
|
||||
case NtPosLook | BeforeChild:
|
||||
// NOTE: the following line causes lookahead/lookbehind to be
|
||||
@@ -317,7 +449,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
w.emit(Forejump)
|
||||
|
||||
case NtOne, NtNotone:
|
||||
w.emit1(InstOp(node.T|ntBits), int(node.Ch))
|
||||
if !w.preconsumed[node] {
|
||||
w.emit1(InstOp(node.T|ntBits), int(node.Ch))
|
||||
}
|
||||
|
||||
case NtNotoneloop, NtNotoneloopatomic, NtNotonelazy, NtOneloop, NtOneloopatomic, NtOnelazy:
|
||||
if node.M > 0 {
|
||||
@@ -348,10 +482,22 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
}
|
||||
|
||||
case NtMulti:
|
||||
w.emit1(InstOp(node.T|ntBits), w.stringCode(node.Str))
|
||||
str := node.Str
|
||||
if w.preconsumed[node] {
|
||||
if bits&Rtl != 0 {
|
||||
str = str[:len(str)-1]
|
||||
} else {
|
||||
str = str[1:]
|
||||
}
|
||||
}
|
||||
if len(str) > 0 {
|
||||
w.emit1(InstOp(node.T|ntBits), w.stringCode(str))
|
||||
}
|
||||
|
||||
case NtSet:
|
||||
w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set))
|
||||
if !w.preconsumed[node] {
|
||||
w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set))
|
||||
}
|
||||
|
||||
case NtRef:
|
||||
w.emit1(InstOp(node.T|ntBits), w.mapCapnum(node.M))
|
||||
@@ -359,6 +505,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary, NtBeginning, NtStart, NtEndZ, NtEnd, NtUpdateBumpalong:
|
||||
w.emit(InstOp(node.T))
|
||||
|
||||
case NtGrapheme:
|
||||
w.emit(InstOp(node.T | ntBits))
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unexpected opcode in regular expression generation: %v", nodetype)
|
||||
}
|
||||
@@ -366,6 +515,17 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *writer) emitCapture(node *RegexNode) bool {
|
||||
if w.quickCaptureSlots == nil {
|
||||
return true
|
||||
}
|
||||
capnum, uncapnum := w.mapCapnum(node.M), w.mapCapnum(node.N)
|
||||
if uncapnum != -1 {
|
||||
return true
|
||||
}
|
||||
return capnum >= 0 && (capnum >= len(w.quickCaptureSlots) || w.quickCaptureSlots[capnum])
|
||||
}
|
||||
|
||||
// To avoid recursion, we use a simple integer stack.
|
||||
// This is the push.
|
||||
func (w *writer) pushInt(i int) {
|
||||
@@ -398,6 +558,53 @@ func (w *writer) patchJump(offset, jumpDest int) {
|
||||
w.emitted[offset+1] = jumpDest
|
||||
}
|
||||
|
||||
func (w *writer) getDispatchCandidates(node *RegexNode) ([]*CharSet, []*RegexNode, []bool, bool) {
|
||||
if info, found := w.dispatchInfo[node]; found {
|
||||
return info.sets, info.leaders, info.complete, info.ok
|
||||
}
|
||||
sets, leaders, complete, ok := node.dispatchCandidates()
|
||||
w.dispatchInfo[node] = dispatchInfo{sets: sets, leaders: leaders, complete: complete, ok: ok}
|
||||
return sets, leaders, complete, ok
|
||||
}
|
||||
|
||||
func (w *writer) dispatchCode(sets []*CharSet) int {
|
||||
if w.counting {
|
||||
return 0
|
||||
}
|
||||
table := DispatchTable{
|
||||
Sets: make([]int, len(sets)),
|
||||
Branches: make([]int, len(sets)),
|
||||
}
|
||||
if len(sets) >= 4 {
|
||||
table.ASCII = &[128]uint16{}
|
||||
} else {
|
||||
table.ASCIIMasks = make([]uint64, len(sets)*2)
|
||||
}
|
||||
for i, set := range sets {
|
||||
table.Sets[i] = w.setCode(set)
|
||||
for ch := rune(0); ch < 128; ch++ {
|
||||
if set.CharIn(ch) {
|
||||
if table.ASCII != nil {
|
||||
table.ASCII[ch] = uint16(i + 1)
|
||||
} else {
|
||||
table.ASCIIMasks[i*2+int(ch>>6)] |= uint64(1) << (ch & 63)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
index := len(w.dispatchtable)
|
||||
w.dispatchtable = append(w.dispatchtable, table)
|
||||
return index
|
||||
}
|
||||
|
||||
func (w *writer) patchDispatchBranch(node *RegexNode, branch, target int) {
|
||||
if w.counting {
|
||||
return
|
||||
}
|
||||
table := w.dispatchCodes[node]
|
||||
w.dispatchtable[table].Branches[branch] = target
|
||||
}
|
||||
|
||||
// Returns an index in the set table for a charset
|
||||
// uses a map to eliminate duplicates.
|
||||
func (w *writer) setCode(set *CharSet) int {
|
||||
|
||||
+26
@@ -237,6 +237,10 @@ func (s *Service) Handlers() (authHandler, avatarHandler http.Handler) {
|
||||
p.Handler(w, r)
|
||||
}
|
||||
|
||||
if s.avatarProxy == nil { // no avatar store configured, avatar route has nothing to serve
|
||||
return withSecurityHeaders(http.HandlerFunc(ah)), withSecurityHeaders(http.NotFoundHandler())
|
||||
}
|
||||
|
||||
return withSecurityHeaders(http.HandlerFunc(ah)), withSecurityHeaders(http.HandlerFunc(s.avatarProxy.Handler))
|
||||
}
|
||||
|
||||
@@ -383,6 +387,28 @@ func (s *Service) AddMicrosoftProvider(cid, csecret, tenant string) {
|
||||
s.addProvider(provider.NewMicrosoft(p))
|
||||
}
|
||||
|
||||
// AddGithubProviderWithNumericID adds github provider deriving the user id from the immutable
|
||||
// numeric account id instead of the login. Logins are released on rename or account removal and
|
||||
// can be claimed by someone else, so an id derived from login may be inherited by the next holder
|
||||
// of the name. This changes the id of every existing github user, see README for the migration note.
|
||||
// If the response carries no usable numeric id the login-derived id is kept.
|
||||
// For advanced configuration (e.g., UserAttributes), construct provider.Params directly.
|
||||
func (s *Service) AddGithubProviderWithNumericID(cid, csecret string) {
|
||||
p := provider.Params{
|
||||
URL: s.opts.URL,
|
||||
JwtService: s.jwtService,
|
||||
Issuer: s.issuer,
|
||||
AvatarSaver: s.avatarProxy,
|
||||
Cid: cid,
|
||||
Csecret: csecret,
|
||||
L: s.logger,
|
||||
UserAttributes: map[string]string{},
|
||||
GithubNumericID: true,
|
||||
AllowedRedirectHosts: s.opts.AllowedRedirectHosts,
|
||||
}
|
||||
s.addProvider(provider.NewGithub(p))
|
||||
}
|
||||
|
||||
// AddDevProvider with a custom host and port
|
||||
func (s *Service) AddDevProvider(host string, port int) {
|
||||
p := provider.Params{
|
||||
|
||||
+74
-16
@@ -43,8 +43,67 @@ func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error
|
||||
}
|
||||
|
||||
avaHash := hash(buf.Bytes(), id)
|
||||
_, err = bucket.UploadFromStream(id+imgSfx, buf, &options.UploadOptions{Metadata: bson.M{"hash": avaHash}})
|
||||
return id + imgSfx, err
|
||||
fileID, err := bucket.UploadFromStream(id+imgSfx, buf, &options.UploadOptions{Metadata: bson.M{"hash": avaHash}})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// gridfs turns every upload with the same name into another revision, so drop what
|
||||
// earlier Put calls left behind. Cleanup runs after the upload, keeping the previous
|
||||
// avatar in place if the write failed, and is best-effort as the new avatar is stored
|
||||
// either way.
|
||||
_ = gf.removeOlderRevisions(bucket, id+imgSfx, fileID)
|
||||
|
||||
return id + imgSfx, nil
|
||||
}
|
||||
|
||||
// removeOlderRevisions deletes the revisions of fileName stored before keepID. Only older
|
||||
// revisions go, so two concurrent Put calls cannot delete each other's upload and leave the
|
||||
// avatar missing; the newer of the two survives.
|
||||
func (gf *GridFS) removeOlderRevisions(bucket *gridfs.Bucket, fileName string, keepID primitive.ObjectID) error {
|
||||
ids, err := gf.revisionIDs(bucket, fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
older := false
|
||||
for _, id := range ids { // newest first, everything past keepID was uploaded earlier
|
||||
if id == keepID {
|
||||
older = true
|
||||
continue
|
||||
}
|
||||
if !older {
|
||||
continue
|
||||
}
|
||||
if e := bucket.Delete(id); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// revisionIDs returns ids of all gridfs files stored under the given name, newest first
|
||||
func (gf *GridFS) revisionIDs(bucket *gridfs.Bucket, fileName string) ([]primitive.ObjectID, error) {
|
||||
sortNewestFirst := options.GridFSFind().SetSort(bson.D{{Key: "uploadDate", Value: -1}, {Key: "_id", Value: -1}})
|
||||
cursor, err := bucket.Find(bson.M{"filename": fileName}, sortNewestFirst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
|
||||
var ids []primitive.ObjectID
|
||||
for cursor.Next(ctx) {
|
||||
r := struct {
|
||||
ID primitive.ObjectID `bson:"_id"`
|
||||
}{}
|
||||
if err = cursor.Decode(&r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
return ids, cursor.Err()
|
||||
}
|
||||
|
||||
// Get avatar reader for avatar id.image
|
||||
@@ -78,7 +137,8 @@ func (gf *GridFS) ID(avatar string) (id string) {
|
||||
if err != nil {
|
||||
return encodeID(avatar)
|
||||
}
|
||||
cursor, err := bucket.Find(bson.M{"filename": avatar})
|
||||
sortNewestFirst := options.GridFSFind().SetSort(bson.D{{Key: "uploadDate", Value: -1}, {Key: "_id", Value: -1}})
|
||||
cursor, err := bucket.Find(bson.M{"filename": avatar}, sortNewestFirst)
|
||||
if err != nil {
|
||||
return encodeID(avatar)
|
||||
}
|
||||
@@ -100,23 +160,21 @@ func (gf *GridFS) Remove(avatar string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cursor, err := bucket.Find(bson.M{"filename": avatar})
|
||||
ids, err := gf.revisionIDs(bucket, avatar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r := struct {
|
||||
ID primitive.ObjectID `bson:"_id"`
|
||||
}{}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
if found := cursor.Next(ctx); found {
|
||||
if err := cursor.Decode(&r); err != nil {
|
||||
return err
|
||||
}
|
||||
return bucket.Delete(r.ID)
|
||||
if len(ids) == 0 {
|
||||
return fmt.Errorf("avatar %s not found: %w", avatar, ErrNotFound)
|
||||
}
|
||||
return fmt.Errorf("avatar %s not found: %w", avatar, ErrNotFound)
|
||||
|
||||
// every revision has to go, deleting the newest one alone leaves the avatar readable
|
||||
for _, id := range ids {
|
||||
if e := bucket.Delete(id); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// List all avatars (ids) on gfs
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ func (fs *LocalFS) Get(avatar string) (reader io.ReadCloser, size int, err error
|
||||
func (fs *LocalFS) ID(avatar string) (id string) {
|
||||
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
|
||||
avFile := path.Join(location, avatar)
|
||||
fi, err := os.Stat(avFile)
|
||||
fi, err := os.Stat(avFile) //nolint:gosec // avatar id is store-generated and validated by the proxy handler
|
||||
if err != nil {
|
||||
return encodeID(avatar)
|
||||
}
|
||||
|
||||
+23
-26
@@ -92,8 +92,9 @@ type AppleHandler struct {
|
||||
// infoURL string not implemented at Apple side
|
||||
endpoint oauth2.Endpoint
|
||||
|
||||
mapUser func(jwt.MapClaims) token.User // map info from InfoURL to User
|
||||
conf AppleConfig // main config for Apple auth provider
|
||||
mapUser func(jwt.MapClaims) token.User // map info from InfoURL to User
|
||||
conf AppleConfig // main config for Apple auth provider
|
||||
jwkCache *appleJWKCache // shared cache of Apple public keys
|
||||
|
||||
PrivateKeyLoader PrivateKeyLoaderInterface // custom function interface for load private key
|
||||
|
||||
@@ -181,6 +182,8 @@ func NewApple(p Params, appleCfg AppleConfig, privateKeyLoader PrivateKeyLoaderI
|
||||
TokenURL: appleTokenURL,
|
||||
},
|
||||
|
||||
jwkCache: &appleJWKCache{},
|
||||
|
||||
mapUser: func(claims jwt.MapClaims) token.User {
|
||||
var usr token.User
|
||||
if uid, ok := claims["sub"]; ok {
|
||||
@@ -261,7 +264,7 @@ func (ah *AppleHandler) LoginHandler(w http.ResponseWriter, r *http.Request) {
|
||||
State: state,
|
||||
From: r.URL.Query().Get("from"),
|
||||
},
|
||||
SessionOnly: r.URL.Query().Get("session") != "" && r.URL.Query().Get("session") != "0",
|
||||
SessionOnly: sessionOnlyFromRequest(r),
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ID: cid,
|
||||
Audience: jwt.ClaimStrings{r.URL.Query().Get("site")},
|
||||
@@ -288,11 +291,11 @@ func (ah *AppleHandler) LoginHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
ah.Logf("[DEBUG] login url %s, claims=%+v", loginURL, claims)
|
||||
|
||||
http.Redirect(w, r, loginURL, http.StatusFound)
|
||||
http.Redirect(w, r, loginURL, http.StatusFound) //nolint:gosec // redirect goes to the fixed apple auth endpoint, request path only affects redirect_uri query param
|
||||
}
|
||||
|
||||
// AuthHandler fills user info and redirects to "from" url. This is callback url redirected locally by browser
|
||||
// GET /callback
|
||||
// AuthHandler fills user info and redirects to "from" url. This is callback url redirected locally by browser.
|
||||
// POST /callback with the default form_post response mode, GET /callback when response mode is overridden
|
||||
func (ah AppleHandler) AuthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// read response form data
|
||||
@@ -338,19 +341,12 @@ func (ah AppleHandler) AuthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// trying to fetch Apple public key (JWK) for verify token signature, it need for verify IDToken received from Apple
|
||||
keySet, err := fetchAppleJWK(r.Context(), ah.conf.jwkURL)
|
||||
if err != nil {
|
||||
ah.Logf("[ERROR] failed to fetch JWK from Apple key service: " + err.Error())
|
||||
rest.SendErrorJSON(w, r, ah.L, http.StatusInternalServerError, nil, fmt.Sprintf("failed to fetch JWK from Apple key service: %s", resp.Error))
|
||||
return
|
||||
}
|
||||
|
||||
// get token claims for extract uid (and email or name if they exist in scope).
|
||||
// jwt v5 parser options enforce iss == https://appleid.apple.com and
|
||||
// the signature is verified with Apple public keys (JWK), served from the handler cache,
|
||||
// while jwt v5 parser options enforce iss == https://appleid.apple.com and
|
||||
// aud == ClientID inline so we don't need a separate validate pass.
|
||||
tokenClaims := jwt.MapClaims{}
|
||||
_, err = jwt.ParseWithClaims(resp.IDToken, tokenClaims, keySet.keyFunc,
|
||||
_, err = jwt.ParseWithClaims(resp.IDToken, tokenClaims, ah.jwkKeyFunc(r.Context()),
|
||||
jwt.WithIssuer(appleIDTokenIssuer),
|
||||
jwt.WithAudience(ah.conf.ClientID))
|
||||
if err != nil {
|
||||
@@ -391,7 +387,7 @@ func (ah AppleHandler) AuthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ID: cid,
|
||||
Audience: oauthClaims.Audience,
|
||||
},
|
||||
SessionOnly: false,
|
||||
SessionOnly: oauthClaims.SessionOnly,
|
||||
AuthProvider: &token.AuthProvider{
|
||||
Name: ah.name,
|
||||
},
|
||||
@@ -411,7 +407,9 @@ func (ah AppleHandler) AuthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
rest.RenderJSON(w, &u)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, oauthClaims.Handshake.From, http.StatusTemporaryRedirect)
|
||||
// see-other makes the browser retrieve the target with GET, so apple's form_post
|
||||
// callback is not replayed as a POST onto the "from" page
|
||||
http.Redirect(w, r, oauthClaims.Handshake.From, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
rest.RenderJSON(w, &u)
|
||||
@@ -461,18 +459,17 @@ func (ah *AppleHandler) exchange(ctx context.Context, code, redirectURI string,
|
||||
return err
|
||||
}
|
||||
|
||||
// trying to decode (unmarshal json) data of response
|
||||
err = json.NewDecoder(res.Body).Decode(result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unmarshalling data from apple service response failed: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err = res.Body.Close(); err != nil {
|
||||
ah.Logf("[ERROR] close request body failed when get access token: %v", err)
|
||||
if e := res.Body.Close(); e != nil {
|
||||
ah.Logf("[ERROR] close request body failed when get access token: %v", e)
|
||||
}
|
||||
}()
|
||||
|
||||
// trying to decode (unmarshal json) data of response
|
||||
if err = json.NewDecoder(res.Body).Decode(result); err != nil {
|
||||
return fmt.Errorf("unmarshalling data from apple service response failed: %w", err)
|
||||
}
|
||||
|
||||
// if above operation done successfully checking a response code and error descriptions, if one exist.
|
||||
// apple service will response either 200 (OK) or 400 (any error).
|
||||
if res.StatusCode != http.StatusOK || result.Error != "" {
|
||||
|
||||
+77
-1
@@ -14,6 +14,7 @@ import (
|
||||
"io"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
@@ -65,17 +66,25 @@ func fetchAppleJWK(ctx context.Context, keyURL string) (set appleKeySet, err err
|
||||
if err != nil {
|
||||
return set, fmt.Errorf("failed to fetch Apple public keys: %w", err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
|
||||
// an error body parses as a key set with no keys, caching it would break every login until it expires
|
||||
if res.StatusCode < http.StatusOK || res.StatusCode >= http.StatusMultipleChoices {
|
||||
return set, fmt.Errorf("failed to fetch Apple public keys, status %s", res.Status)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return set, fmt.Errorf("failed read data after Apple public key fetched: %w", err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
|
||||
set, err = parseAppleJWK(data)
|
||||
if err != nil {
|
||||
return set, fmt.Errorf("get set of apple public key failed: %w", err)
|
||||
}
|
||||
if len(set.keys) == 0 {
|
||||
return appleKeySet{}, fmt.Errorf("no keys in Apple public key response")
|
||||
}
|
||||
|
||||
return set, nil
|
||||
}
|
||||
@@ -176,3 +185,70 @@ func (aks *appleKeySet) keyFunc(token *jwt.Token) (any, error) {
|
||||
|
||||
return key.publicKey, nil
|
||||
}
|
||||
|
||||
// appleJWKTTL is how long a successfully fetched key set is reused before a refresh
|
||||
const appleJWKTTL = time.Hour
|
||||
|
||||
// appleJWKStaleTTL bounds the use of a cached key set when the key service is unreachable
|
||||
const appleJWKStaleTTL = 12 * time.Hour
|
||||
|
||||
// appleJWKCache keeps the last fetched Apple key set. AppleHandler is copied by value on
|
||||
// every request, so the cache is held behind a pointer and shared by all copies.
|
||||
type appleJWKCache struct {
|
||||
lock sync.Mutex
|
||||
set appleKeySet
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
// jwkSet returns a set of Apple public keys able to verify kid. The cached set is reused
|
||||
// until appleJWKTTL passes; an unknown kid forces a refresh to pick up Apple's key rotation.
|
||||
// If the refresh fails, a cached set holding the kid is used for up to appleJWKStaleTTL, so
|
||||
// logins survive a short outage of the key service.
|
||||
func (ah AppleHandler) jwkSet(ctx context.Context, kid string) (appleKeySet, error) {
|
||||
if ah.jwkCache == nil { // handler constructed without NewApple
|
||||
return fetchAppleJWK(ctx, ah.conf.jwkURL)
|
||||
}
|
||||
|
||||
ah.jwkCache.lock.Lock()
|
||||
defer ah.jwkCache.lock.Unlock()
|
||||
|
||||
cached := ah.jwkCache.set
|
||||
_, cachedErr := cached.get(kid)
|
||||
if cachedErr == nil && time.Since(ah.jwkCache.fetchedAt) < appleJWKTTL {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
set, err := fetchAppleJWK(ctx, ah.conf.jwkURL)
|
||||
if err != nil {
|
||||
if cachedErr == nil && time.Since(ah.jwkCache.fetchedAt) < appleJWKStaleTTL {
|
||||
ah.Logf("[WARN] failed to refresh Apple public keys, using cached set: %v", err)
|
||||
return cached, nil
|
||||
}
|
||||
return set, err
|
||||
}
|
||||
|
||||
ah.jwkCache.set, ah.jwkCache.fetchedAt = set, time.Now()
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// jwkKeyFunc verifies an Apple id_token signature with the key named by the token's kid header
|
||||
func (ah AppleHandler) jwkKeyFunc(ctx context.Context) jwt.Keyfunc {
|
||||
return func(jwtToken *jwt.Token) (any, error) {
|
||||
kid, ok := jwtToken.Header["kid"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("get JWT kid header not found")
|
||||
}
|
||||
|
||||
set, err := ah.jwkSet(ctx, kid)
|
||||
if err != nil {
|
||||
ah.Logf("[ERROR] failed to fetch JWK from Apple key service: %v", err)
|
||||
return nil, fmt.Errorf("failed to fetch JWK from Apple key service: %w", err)
|
||||
}
|
||||
|
||||
key, err := set.get(kid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key.publicKey, nil
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -193,7 +193,8 @@ func (c *CustomServer) handleAvatar(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, err = w.Write(b); err != nil {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
if _, err = w.Write(b); err != nil { //nolint:gosec // generated identicon png, not reflected markup
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
+2
-1
@@ -130,7 +130,8 @@ func (d *DevAuthServer) Run(ctx context.Context) { // nolint (gocyclo)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, err = w.Write(b); err != nil {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
if _, err = w.Write(b); err != nil { //nolint:gosec // generated identicon png, not reflected markup
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
+3
-3
@@ -60,9 +60,9 @@ func (p DirectHandler) Name() string { return p.ProviderName }
|
||||
|
||||
// LoginHandler checks "user" and "passwd" against data store and makes jwt if all passed.
|
||||
//
|
||||
// GET /something?user=name&passwd=xyz&aud=bar&sess=[0|1]
|
||||
// GET /something?user=name&passwd=xyz&aud=bar&session=[0|1]
|
||||
//
|
||||
// POST /something?sess[0|1]
|
||||
// POST /something?session=[0|1]
|
||||
// Accepts application/x-www-form-urlencoded or application/json encoded requests.
|
||||
//
|
||||
// application/x-www-form-urlencoded body example:
|
||||
@@ -82,7 +82,7 @@ func (p DirectHandler) LoginHandler(w http.ResponseWriter, r *http.Request) {
|
||||
rest.SendErrorJSON(w, logReq, p.L, http.StatusBadRequest, err, "failed to parse credentials")
|
||||
return
|
||||
}
|
||||
sessOnly := r.URL.Query().Get("sess") == "1"
|
||||
sessOnly := sessionOnlyFromRequest(r)
|
||||
if p.CredChecker == nil {
|
||||
rest.SendErrorJSON(w, logReq, p.L, http.StatusInternalServerError,
|
||||
fmt.Errorf("no credential checker"), "no credential checker")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user