Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Umputun
|
dfb3436f30 |
+3
-16
@@ -1,26 +1,15 @@
|
||||
/logs/
|
||||
/target/
|
||||
/var/
|
||||
/frontend/node_modules/
|
||||
/frontend/public/
|
||||
/.vscode/
|
||||
/.idea/
|
||||
/bin/
|
||||
/.git/
|
||||
|
||||
# frontend files not needed in docker image
|
||||
/frontend/node_modules/
|
||||
/frontend/apps/remark42/node_modules/
|
||||
/frontend/apps/remark42/public/
|
||||
|
||||
# source files
|
||||
docker-compose.yml
|
||||
compose-dev-backend.yml
|
||||
compose-dev-frontend.yml
|
||||
compose-private-backend.yml
|
||||
compose-private-frontend.yml
|
||||
compose-e2e-test.yml
|
||||
compose-private.yml
|
||||
rest-client.env.json
|
||||
Makefile
|
||||
|
||||
# generated files
|
||||
*.cov
|
||||
@@ -32,6 +21,4 @@ debug.test
|
||||
*.test
|
||||
remark42
|
||||
/backend/var/
|
||||
|
||||
# go e2e suite, never built into the image
|
||||
/e2e/
|
||||
compose-private-backend.yml
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
kind: pipeline
|
||||
name: default
|
||||
type: docker
|
||||
|
||||
steps:
|
||||
|
||||
- name: build server
|
||||
image: umputun/baseimage:buildgo-latest
|
||||
commands:
|
||||
- cd backend/app
|
||||
- go build -mod=vendor
|
||||
- echo "build completed"
|
||||
|
||||
- name: docker master
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: umputun/remark42
|
||||
username:
|
||||
from_secret: docker_username
|
||||
password:
|
||||
from_secret: docker_password
|
||||
build_args:
|
||||
- DRONE=${DRONE}
|
||||
- DRONE_TAG=${DRONE_TAG}
|
||||
- DRONE_COMMIT=${DRONE_COMMIT}
|
||||
- DRONE_BRANCH=${DRONE_BRANCH}
|
||||
tags:
|
||||
- ${DRONE_COMMIT_BRANCH/\//-}
|
||||
when:
|
||||
branch: [master]
|
||||
event: push
|
||||
|
||||
- name: docker tag
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: umputun/remark42
|
||||
username:
|
||||
from_secret: docker_username
|
||||
password:
|
||||
from_secret: docker_password
|
||||
build_args:
|
||||
- DRONE=${DRONE}
|
||||
- DRONE_TAG=${DRONE_TAG}
|
||||
- DRONE_COMMIT=${DRONE_COMMIT}
|
||||
tags:
|
||||
- ${DRONE_TAG}
|
||||
- latest
|
||||
when:
|
||||
event: tag
|
||||
|
||||
- name: docker branch
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: umputun/remark42
|
||||
username:
|
||||
from_secret: docker_username
|
||||
password:
|
||||
from_secret: docker_password
|
||||
build_args:
|
||||
- DRONE=${DRONE}
|
||||
- DRONE_COMMIT=${DRONE_COMMIT}
|
||||
- DRONE_BRANCH=${DRONE_BRANCH}
|
||||
tags:
|
||||
- ${DRONE_COMMIT_BRANCH/\//-}
|
||||
dry_run: true
|
||||
when:
|
||||
branch:
|
||||
exclude: [master, release/*]
|
||||
event: push
|
||||
|
||||
- name: artifacts tag
|
||||
image: plugins/docker
|
||||
settings:
|
||||
dockerfile: Dockerfile.artifacts
|
||||
build_args:
|
||||
- DRONE=${DRONE}
|
||||
- DRONE_TAG=${DRONE_TAG}
|
||||
- DRONE_COMMIT=${DRONE_COMMIT}
|
||||
- GITHUB_TOKEN=${GITHUB_TOKEN}
|
||||
when:
|
||||
event: tag
|
||||
|
||||
- name: deploy
|
||||
image: docker.umputun.com/system/deploy-ci:master
|
||||
commands:
|
||||
- ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
|
||||
- ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
|
||||
when:
|
||||
branch: master
|
||||
event: push
|
||||
|
||||
- name: notify
|
||||
image: drillster/drone-email
|
||||
settings:
|
||||
host: smtp.mailgun.org
|
||||
username:
|
||||
from_secret: email_username
|
||||
password:
|
||||
from_secret: email_password
|
||||
from: drone@mg.umputun.dev
|
||||
recipients: [ sys@umputun.dev ]
|
||||
when:
|
||||
status: [ changed, failure ]
|
||||
@@ -1,13 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
insert_final_newline = true
|
||||
|
||||
[*.md]
|
||||
indent_style = space
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.{yml,json}]
|
||||
indent_size = 2
|
||||
indent_style = space
|
||||
+1
-2
@@ -2,5 +2,4 @@
|
||||
# Unless a later match takes precedence, @umputun will be requested for
|
||||
# review when someone opens a pull request.
|
||||
|
||||
* @umputun
|
||||
frontend/* @umputun @akellbl4 @Mavrin
|
||||
* @umputun
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
|
||||
# npm updates are switched off entirely. open-pull-requests-limit bounds version
|
||||
# updates only, so the ignore entries below are what also stops security updates;
|
||||
# removing the npm entries would not work, as security updates come from alerts
|
||||
# rather than from this file.
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"GitHub Actions updates":
|
||||
patterns:
|
||||
- "*"
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/backend"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"Go modules updates":
|
||||
dependency-type: "production"
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/e2e"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"Go modules updates":
|
||||
dependency-type: "production"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend"
|
||||
open-pull-requests-limit: 0
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"NPM modules updates":
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/apps/remark42"
|
||||
open-pull-requests-limit: 0
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"NPM modules updates":
|
||||
dependency-type: "production"
|
||||
"NPM modules updates for tests":
|
||||
dependency-type: "development"
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/site"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
"Site image updates":
|
||||
patterns:
|
||||
- "*"
|
||||
@@ -1,117 +0,0 @@
|
||||
name: backend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
tags:
|
||||
paths:
|
||||
- ".github/workflows/ci-backend.yml"
|
||||
- "backend/**"
|
||||
- "Dockerfile"
|
||||
- "docker-init.sh"
|
||||
- ".dockerignore"
|
||||
- "!backend/scripts/**"
|
||||
- "!**.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-backend.yml"
|
||||
- "backend/**"
|
||||
- "Dockerfile"
|
||||
- "docker-init.sh"
|
||||
- ".dockerignore"
|
||||
- "!backend/scripts/**"
|
||||
- "!**.md"
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test & Coverage
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: debug if needed
|
||||
run: if [[ "$DEBUG" == "true" ]]; then env; fi
|
||||
env:
|
||||
DEBUG: ${{secrets.DEBUG}}
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend
|
||||
|
||||
- name: test and build backend
|
||||
run: |
|
||||
go test -race -timeout=300s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
|
||||
cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
|
||||
go build -race ./...
|
||||
working-directory: backend/app
|
||||
env:
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: test examples
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build -race ./...
|
||||
working-directory: backend/_example/memory_store
|
||||
env:
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: "v2.13.1"
|
||||
working-directory: backend/app
|
||||
|
||||
- name: golangci-lint on example directory
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: "v2.13.1"
|
||||
args: --config ../../.golangci.yml
|
||||
working-directory: backend/_example/memory_store
|
||||
|
||||
- name: submit coverage
|
||||
run: |
|
||||
go install github.com/mattn/goveralls@latest
|
||||
goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
|
||||
working-directory: backend
|
||||
env:
|
||||
COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
vulncheck:
|
||||
name: Vulnerability scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
# both go.sum files so the cache key covers the main and example modules scanned below
|
||||
cache-dependency-path: |
|
||||
backend/go.sum
|
||||
backend/_example/memory_store/go.sum
|
||||
|
||||
- name: govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.5.0
|
||||
govulncheck ./...
|
||||
(cd _example/memory_store && govulncheck ./...)
|
||||
working-directory: backend
|
||||
env:
|
||||
# ignore the committed vendor dirs and resolve modules from the cache so
|
||||
# both the main module and the nested example module scan consistently
|
||||
GOFLAGS: "-mod=readonly"
|
||||
@@ -1,63 +1,31 @@
|
||||
name: build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
tags:
|
||||
paths:
|
||||
- '.github/workflows/ci-build.yml'
|
||||
- 'backend/**'
|
||||
- 'frontend/**'
|
||||
- '.dockerignore'
|
||||
- 'docker-init.sh'
|
||||
- 'Dockerfile'
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-build.yml"
|
||||
- "backend/**"
|
||||
- "frontend/apps/**"
|
||||
- ".dockerignore"
|
||||
- "docker-init.sh"
|
||||
- "Dockerfile"
|
||||
- "!**.md"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
- '.github/workflows/ci-build.yml'
|
||||
- 'backend/**'
|
||||
- 'frontend/**'
|
||||
- '.dockerignore'
|
||||
- 'docker-init.sh'
|
||||
- 'Dockerfile'
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
name: Validate Docker build
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: free disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
docker system prune -af
|
||||
|
||||
- name: build docker image without pushing
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
cache-from: type=gha,scope=main
|
||||
cache-to: type=gha,scope=main,mode=max,ignore-error=true
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
|
||||
- name: build example docker image without pushing
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: backend/_example/memory_store/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
cache-from: type=gha,scope=example
|
||||
cache-to: type=gha,scope=example,mode=max,ignore-error=true
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
- name: build docker image
|
||||
run: docker build --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github .
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
name: compose
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-compose.yml"
|
||||
- "**compose*.yml"
|
||||
- "**compose*.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-compose.yml"
|
||||
- "**compose*.yml"
|
||||
- "**compose*.yaml"
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate compose files
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: validate tracked compose files
|
||||
run: |
|
||||
set -euo pipefail
|
||||
n=0
|
||||
# null-delimited to stay safe with unusual filenames; exclude this
|
||||
# workflow (its name contains "compose") and vendored compose files.
|
||||
# filenames are not echoed as workflow commands to avoid log-command injection
|
||||
while IFS= read -r -d '' f; do
|
||||
docker compose -f "$f" config --quiet
|
||||
n=$((n + 1))
|
||||
done < <(git ls-files -z '*compose*.yml' '*compose*.yaml' ':!:*/vendor/*' ':!:.github/*')
|
||||
if [ "$n" -eq 0 ]; then
|
||||
echo "no compose files found" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "validated $n compose file(s)"
|
||||
@@ -1,37 +0,0 @@
|
||||
name: docs versions
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-docs-versions.yml"
|
||||
- "scripts/check-documented-versions.sh"
|
||||
- "site/content/docs/getting-started/installation/index.md"
|
||||
- "backend/go.mod"
|
||||
- "frontend/apps/remark42/package.json"
|
||||
- "frontend/.nvmrc"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-docs-versions.yml"
|
||||
- "scripts/check-documented-versions.sh"
|
||||
- "site/content/docs/getting-started/installation/index.md"
|
||||
- "backend/go.mod"
|
||||
- "frontend/apps/remark42/package.json"
|
||||
- "frontend/.nvmrc"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Documented versions
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check documented versions against the repository
|
||||
run: ./scripts/check-documented-versions.sh
|
||||
@@ -1,195 +0,0 @@
|
||||
name: frontend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend.yml"
|
||||
- "frontend/**"
|
||||
- "!**.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-frontend.yml"
|
||||
- "frontend/**"
|
||||
- "!**.md"
|
||||
|
||||
jobs:
|
||||
translations-check:
|
||||
name: Translations check
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Translations check
|
||||
run: pnpm translation-check
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
type-check:
|
||||
name: Type check
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Run type check
|
||||
run: pnpm type-check
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
lint:
|
||||
name: Eslint & Stylelint
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Run linters
|
||||
run: pnpm lint
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
size-limit:
|
||||
name: Size limit
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
env:
|
||||
CI_JOB_NUMBER: 1
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Check bundle size
|
||||
uses: andresz1/size-limit-action@94bc357df29c36c8f8d50ea497c3e225c3c95d1d
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
directory: ./frontend/apps/remark42
|
||||
package_manager: pnpm
|
||||
|
||||
test:
|
||||
name: Tests & Coverage
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
node: [24]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Test & Coverage
|
||||
run: pnpm coverage
|
||||
working-directory: ./frontend/apps/remark42
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v7
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
working-directory: ./frontend/apps/remark42
|
||||
codecov_yml_path: ./frontend/apps/remark42/codecov.yml
|
||||
@@ -1,175 +0,0 @@
|
||||
name: site
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- ".github/workflows/ci-site.yml"
|
||||
- "site/**"
|
||||
- "!**/CLAUDE.md"
|
||||
- "!site/README.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/ci-site.yml"
|
||||
- "site/**"
|
||||
- "!**/CLAUDE.md"
|
||||
- "!site/README.md"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Build site image (pull request)
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: set up docker buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: build image without pushing
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./site
|
||||
load: true
|
||||
push: false
|
||||
cache-from: |
|
||||
type=gha,scope=site-pr
|
||||
type=gha,scope=site-linux/amd64
|
||||
cache-to: type=gha,scope=site-pr,mode=max,ignore-error=true
|
||||
|
||||
build:
|
||||
name: Build site image (${{ matrix.platform }})
|
||||
if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
artifact: linux-amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
artifact: linux-arm64
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: login to ghcr.io
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.PKG_TOKEN }}
|
||||
|
||||
- name: build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./site
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=site-${{ matrix.platform }}
|
||||
cache-to: type=gha,scope=site-${{ matrix.platform }},mode=max,ignore-error=true
|
||||
outputs: type=image,name=ghcr.io/umputun/remark42-site,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: upload digest
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: site-digests-${{ matrix.artifact }}
|
||||
path: /tmp/digests/*
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
name: Create site multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: download digests
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: site-digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: verify all digests present
|
||||
run: |
|
||||
expected=2
|
||||
actual=$(find /tmp/digests -maxdepth 1 -type f | wc -l)
|
||||
if [ "$actual" -ne "$expected" ]; then
|
||||
echo "Expected $expected digests, found $actual"
|
||||
ls -la /tmp/digests
|
||||
exit 1
|
||||
fi
|
||||
echo "All $expected digests present"
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: login to ghcr.io
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.PKG_TOKEN }}
|
||||
|
||||
- name: create manifest and push
|
||||
working-directory: /tmp/digests
|
||||
env:
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
run: |
|
||||
ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
|
||||
if [[ "$GITHUB_REF" == refs/tags/* ]]; then
|
||||
docker buildx imagetools create \
|
||||
-t ghcr.io/umputun/remark42-site:${ref} \
|
||||
-t ghcr.io/umputun/remark42-site:latest \
|
||||
$(printf 'ghcr.io/umputun/remark42-site@sha256:%s ' *)
|
||||
else
|
||||
docker buildx imagetools create \
|
||||
-t ghcr.io/umputun/remark42-site:${ref} \
|
||||
$(printf 'ghcr.io/umputun/remark42-site@sha256:%s ' *)
|
||||
fi
|
||||
|
||||
deploy:
|
||||
name: Deploy site
|
||||
runs-on: ubuntu-latest
|
||||
needs: merge
|
||||
if: github.ref == 'refs/heads/master' || github.event_name == 'release'
|
||||
permissions: {} # only calls an external URL via curl, no GitHub API access needed
|
||||
|
||||
steps:
|
||||
- name: trigger deployment
|
||||
env:
|
||||
UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
|
||||
run: curl -sf https://jess.umputun.com/update/remark42-site/${UPDATER_KEY}
|
||||
@@ -0,0 +1,63 @@
|
||||
name: test_backend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
tags:
|
||||
paths:
|
||||
- '.github/workflows/ci-test-backend.yml'
|
||||
- 'backend/**'
|
||||
- '!backend/scripts/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/ci-test-backend.yml'
|
||||
- 'backend/**'
|
||||
- '!backend/scripts/**'
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: debug if needed
|
||||
run: if [[ "$DEBUG" == "true" ]]; then env; fi
|
||||
env:
|
||||
DEBUG: ${{secrets.DEBUG}}
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.14
|
||||
|
||||
- name: install golangci-lint and goveralls
|
||||
run: |
|
||||
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.25.0
|
||||
go get -u github.com/mattn/goveralls
|
||||
|
||||
- name: test and lint backend
|
||||
run: |
|
||||
go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
|
||||
cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
|
||||
$GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
|
||||
working-directory: backend/app
|
||||
env:
|
||||
GOFLAGS: "-mod=vendor"
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: test and lint examples
|
||||
run: |
|
||||
go version
|
||||
$GITHUB_WORKSPACE/golangci-lint version
|
||||
go test -race ./...
|
||||
$GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
|
||||
working-directory: backend/_example/memory_store
|
||||
env:
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: submit coverage
|
||||
run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
|
||||
working-directory: backend
|
||||
env:
|
||||
COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,28 @@
|
||||
name: test_frontend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
tags:
|
||||
paths:
|
||||
- '.github/workflows/ci-test-frontend.yml'
|
||||
- 'frontend/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/ci-test-frontend.yml'
|
||||
- 'frontend/**'
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-node@v1
|
||||
|
||||
- run: npm ci --loglevel warn
|
||||
working-directory: ./frontend
|
||||
|
||||
- run: npx run-p check lint
|
||||
working-directory: ./frontend
|
||||
@@ -1,215 +0,0 @@
|
||||
name: docker
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [backend, frontend]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build Docker image (${{ matrix.platform }})
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event != 'pull_request' &&
|
||||
(github.event.workflow_run.head_branch == 'master' ||
|
||||
startsWith(github.event.workflow_run.head_branch, 'v'))
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
artifact: linux-amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
artifact: linux-arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: login to ghcr.io
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.PKG_TOKEN }}
|
||||
|
||||
- name: login to DockerHub
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: umputun
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: free disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
docker system prune -af
|
||||
|
||||
- name: build and push to ghcr.io by digest
|
||||
id: build-ghcr
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||
cache-to: type=gha,scope=${{ matrix.platform }},mode=max
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
CI=github
|
||||
GITHUB_SHA=${{ github.event.workflow_run.head_sha }}
|
||||
GIT_BRANCH=${{ github.event.workflow_run.head_branch }}
|
||||
GITHUB_REF=refs/heads/${{ github.event.workflow_run.head_branch }}
|
||||
outputs: type=image,name=ghcr.io/umputun/remark42,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: build and push to DockerHub by digest
|
||||
id: build-dockerhub
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||
build-args: |
|
||||
SKIP_BACKEND_TEST=true
|
||||
SKIP_FRONTEND_TEST=true
|
||||
CI=github
|
||||
GITHUB_SHA=${{ github.event.workflow_run.head_sha }}
|
||||
GIT_BRANCH=${{ github.event.workflow_run.head_branch }}
|
||||
GITHUB_REF=refs/heads/${{ github.event.workflow_run.head_branch }}
|
||||
outputs: type=image,name=umputun/remark42,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: export digests
|
||||
run: |
|
||||
mkdir -p /tmp/digests/ghcr /tmp/digests/dockerhub
|
||||
digest_ghcr="${{ steps.build-ghcr.outputs.digest }}"
|
||||
digest_dockerhub="${{ steps.build-dockerhub.outputs.digest }}"
|
||||
touch "/tmp/digests/ghcr/${digest_ghcr#sha256:}"
|
||||
touch "/tmp/digests/dockerhub/${digest_dockerhub#sha256:}"
|
||||
|
||||
- name: upload ghcr digest
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digests-ghcr-${{ matrix.artifact }}
|
||||
path: /tmp/digests/ghcr/*
|
||||
retention-days: 1
|
||||
|
||||
- name: upload dockerhub digest
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digests-dockerhub-${{ matrix.artifact }}
|
||||
path: /tmp/digests/dockerhub/*
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
name: Create multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: download ghcr digests
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: /tmp/digests/ghcr
|
||||
pattern: digests-ghcr-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: download dockerhub digests
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: /tmp/digests/dockerhub
|
||||
pattern: digests-dockerhub-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: verify all digests present
|
||||
run: |
|
||||
expected=2
|
||||
for registry in ghcr dockerhub; do
|
||||
actual=$(find /tmp/digests/$registry -maxdepth 1 -type f | wc -l)
|
||||
if [ "$actual" -ne "$expected" ]; then
|
||||
echo "Expected $expected digests for $registry, found $actual"
|
||||
ls -la /tmp/digests/$registry
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "All digests present for both registries"
|
||||
|
||||
- name: set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: login to ghcr.io
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.PKG_TOKEN }}
|
||||
|
||||
- name: login to DockerHub
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: umputun
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: create ghcr.io manifest and push
|
||||
working-directory: /tmp/digests/ghcr
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
if [[ "$HEAD_BRANCH" == v* ]]; then
|
||||
docker buildx imagetools create \
|
||||
-t ghcr.io/umputun/remark42:${HEAD_BRANCH} \
|
||||
-t ghcr.io/umputun/remark42:latest \
|
||||
$(printf 'ghcr.io/umputun/remark42@sha256:%s ' *)
|
||||
else
|
||||
docker buildx imagetools create \
|
||||
-t ghcr.io/umputun/remark42:${HEAD_BRANCH} \
|
||||
$(printf 'ghcr.io/umputun/remark42@sha256:%s ' *)
|
||||
fi
|
||||
|
||||
- name: create DockerHub manifest and push
|
||||
working-directory: /tmp/digests/dockerhub
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
if [[ "$HEAD_BRANCH" == v* ]]; then
|
||||
docker buildx imagetools create \
|
||||
-t umputun/remark42:${HEAD_BRANCH} \
|
||||
-t umputun/remark42:latest \
|
||||
$(printf 'umputun/remark42@sha256:%s ' *)
|
||||
else
|
||||
docker buildx imagetools create \
|
||||
-t umputun/remark42:${HEAD_BRANCH} \
|
||||
$(printf 'umputun/remark42@sha256:%s ' *)
|
||||
fi
|
||||
|
||||
deploy:
|
||||
name: Deploy to remark42.com
|
||||
runs-on: ubuntu-latest
|
||||
needs: merge
|
||||
if: github.event.workflow_run.head_branch == 'master'
|
||||
permissions: {} # only calls an external URL via curl, no GitHub API access needed
|
||||
|
||||
steps:
|
||||
- name: trigger deployment
|
||||
env:
|
||||
UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
|
||||
run: curl -sf https://jess.umputun.com/update/remark42-core/${UPDATER_KEY}
|
||||
@@ -1,125 +0,0 @@
|
||||
name: e2e
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- ".github/workflows/e2e-tests.yml"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "e2e/**"
|
||||
- "compose-e2e-test.yml"
|
||||
- "Dockerfile"
|
||||
- "!**.md"
|
||||
|
||||
pull_request:
|
||||
branches: [master]
|
||||
paths:
|
||||
- ".github/workflows/e2e-tests.yml"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "e2e/**"
|
||||
- "compose-e2e-test.yml"
|
||||
- "Dockerfile"
|
||||
- "!**.md"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# cheap gate: catches a compile break or a lint regression in the build-tagged suite
|
||||
# without paying for the docker build and the browser download
|
||||
vet:
|
||||
name: Vet
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: e2e/go.mod
|
||||
cache-dependency-path: e2e/go.sum
|
||||
|
||||
- name: Vet
|
||||
run: cd e2e && go vet -tags=e2e ./...
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.13.1
|
||||
working-directory: e2e
|
||||
args: --build-tags=e2e --config ../backend/.golangci.yml
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
needs: vet
|
||||
# generous against the docker build plus one 8m go test: a job cancelled on timeout skips
|
||||
# its own failure steps, so the run would end with neither logs nor traces
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: e2e/go.mod
|
||||
cache-dependency-path: e2e/go.sum
|
||||
|
||||
# two directories: the driver (node plus the npm package) and the browser builds,
|
||||
# which include firefox and webkit for the rendering tests
|
||||
- name: Cache playwright driver and browsers
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
~/.cache/ms-playwright-go
|
||||
key: playwright-${{ hashFiles('e2e/go.sum') }}
|
||||
restore-keys: playwright-
|
||||
|
||||
# E2E_STAMP is what the suite compares the running stack against, so a stack started here
|
||||
# has to carry the same value `make e2e-up` and the suite itself would give it
|
||||
- name: Build & start the stack
|
||||
run: |
|
||||
./e2e/tls/generate.sh
|
||||
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 E2E_STAMP=$(./e2e/stamp.sh) \
|
||||
docker compose -f compose-e2e-test.yml up -d --build --quiet-pull --wait
|
||||
|
||||
# no retry: a failure here is evidence about a suite too young to have a flake rate,
|
||||
# and a rerun is how an intermittent regression becomes invisible. revisit when there
|
||||
# are failures on record to look at
|
||||
- name: Run e2e
|
||||
# stamps this run's comment threads with the CI run, so a thread url in a trace or a
|
||||
# log names the run it came from
|
||||
env:
|
||||
E2E_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# 20m, matching the Makefile. the suite runs about four minutes on a laptop and a runner
|
||||
# is slower, so a tighter budget turns a loaded runner into a timeout panic instead of a
|
||||
# readable failure. the job's own timeout above is what bounds a wedged run
|
||||
run: cd e2e && go test -tags=e2e -count 1 -timeout 20m -v ./...
|
||||
|
||||
- name: Server logs on failure
|
||||
if: failure()
|
||||
run: docker compose -f compose-e2e-test.yml logs --tail=200
|
||||
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: playwright-traces
|
||||
path: e2e/traces/
|
||||
retention-days: 30
|
||||
if-no-files-found: ignore
|
||||
@@ -1,145 +0,0 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/release.yml"
|
||||
- ".goreleaser.yml"
|
||||
- "Makefile"
|
||||
- "scripts/**"
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "!backend/**.md"
|
||||
- "!frontend/**.md"
|
||||
- "README.md"
|
||||
- "LICENSE"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 24
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: test and build backend
|
||||
run: |
|
||||
go test -race -timeout=300s ./...
|
||||
go build -race ./...
|
||||
working-directory: backend/app
|
||||
env:
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: test examples
|
||||
run: |
|
||||
go test -race ./...
|
||||
go build -race ./...
|
||||
working-directory: backend/_example/memory_store
|
||||
env:
|
||||
TZ: "America/Chicago"
|
||||
|
||||
- name: install frontend dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: frontend/apps/remark42
|
||||
env:
|
||||
CI: "true"
|
||||
|
||||
- name: check frontend
|
||||
run: |
|
||||
pnpm lint
|
||||
pnpm type-check
|
||||
pnpm test --runInBand
|
||||
working-directory: frontend/apps/remark42
|
||||
env:
|
||||
CI: "true"
|
||||
|
||||
- name: check goreleaser snapshot
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
with:
|
||||
version: latest
|
||||
args: release --snapshot --clean --skip=publish
|
||||
env:
|
||||
SKIP_PNPM_INSTALL: "true"
|
||||
|
||||
- name: clean generated release assets
|
||||
if: always()
|
||||
run: ./scripts/cleanup-release-assets.sh
|
||||
|
||||
release:
|
||||
if: github.event_name == 'push'
|
||||
needs: validate
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: install go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: "1.25"
|
||||
check-latest: true
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: install pnpm
|
||||
uses: pnpm/action-setup@v6.0.10
|
||||
with:
|
||||
version: 10.10.0
|
||||
run_install: false
|
||||
|
||||
- name: install node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 24
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: frontend/apps/remark42/pnpm-lock.yaml
|
||||
|
||||
- name: install frontend dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: frontend/apps/remark42
|
||||
env:
|
||||
CI: "true"
|
||||
|
||||
- name: run goreleaser
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
with:
|
||||
version: latest
|
||||
args: release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SKIP_PNPM_INSTALL: "true"
|
||||
|
||||
- name: clean generated release assets
|
||||
if: always()
|
||||
run: ./scripts/cleanup-release-assets.sh
|
||||
+3
-14
@@ -8,6 +8,9 @@ debug
|
||||
debug.test
|
||||
.vscode
|
||||
.idea/
|
||||
/frontend/node_modules/
|
||||
/frontend/public/
|
||||
/frontend/coverage
|
||||
*.prof
|
||||
*.test
|
||||
/rest-client.env.json
|
||||
@@ -15,23 +18,9 @@ debug.test
|
||||
.mongo
|
||||
remark42
|
||||
/bin/
|
||||
/dist/
|
||||
/backend/var/
|
||||
/backend/app/var/
|
||||
/backend/app/cmd/web/
|
||||
/backend/*.html.tmpl
|
||||
compose-private-backend.yml
|
||||
compose-private-frontend.yml
|
||||
compose-private.yml
|
||||
/backend/_example/*/vendor
|
||||
http-client.env.json
|
||||
/backend/app/cmd/var
|
||||
|
||||
# ralphex progress logs
|
||||
.ralphex/progress/
|
||||
|
||||
# traces from failed e2e runs
|
||||
/e2e/traces/
|
||||
|
||||
# self-signed certificate for the e2e https services, made by e2e/tls/generate.sh
|
||||
/e2e/tls/*.pem
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
version: 2
|
||||
|
||||
project_name: remark42
|
||||
|
||||
git:
|
||||
ignore_tags:
|
||||
- backend/*
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- ./scripts/prepare-release-assets.sh
|
||||
|
||||
builds:
|
||||
- id: remark42
|
||||
dir: backend
|
||||
main: ./app
|
||||
binary: "remark42.{{ .Os }}-{{ .Arch }}"
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
- freebsd
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
- "386"
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: "386"
|
||||
- goos: freebsd
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: "386"
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: "386"
|
||||
ldflags:
|
||||
- -s -w -X main.revision={{ .Tag }}-{{ .ShortCommit }}-{{ trimsuffix (replace (replace .CommitDate "-" "") ":" "") "Z" }}
|
||||
|
||||
archives:
|
||||
- id: remark42
|
||||
ids:
|
||||
- remark42
|
||||
name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}"
|
||||
formats:
|
||||
- tar.gz
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats:
|
||||
- zip
|
||||
files:
|
||||
- LICENSE
|
||||
- README.md
|
||||
|
||||
release:
|
||||
name_template: "Version {{ .Version }}"
|
||||
mode: keep-existing
|
||||
@@ -1,100 +0,0 @@
|
||||
# Remark42 Development Guidelines
|
||||
|
||||
## Build/Test/Lint Commands
|
||||
- **Backend**:
|
||||
- Run server: `make rundev`
|
||||
- Build: `make backend`
|
||||
- Race test: `make race_test`
|
||||
- **Backend Testing**:
|
||||
- Run all tests: `cd backend/app && go test -timeout=300s -count 1 ./...`
|
||||
- Run single test: `cd backend/app && go test -run TestName ./path/to/package`
|
||||
- **IMPORTANT**: Run example tests: `cd backend/_example/memory_store && go test -race ./... && go build -race ./...`
|
||||
- **Frontend**:
|
||||
- Development: `cd frontend/apps/remark42 && pnpm dev`
|
||||
- Tests: `cd frontend/apps/remark42 && pnpm test`
|
||||
- **End-to-end**: `make e2e` drives the widget in a real browser; see `e2e/README.md`. Build-tagged, so `go test ./...` never runs it.
|
||||
- **Lint**:
|
||||
- Backend: `cd backend && golangci-lint run`
|
||||
- **IMPORTANT**: Example lint: `cd backend/_example/memory_store && golangci-lint run --config ../../.golangci.yml`
|
||||
- Frontend: `cd frontend/apps/remark42 && pnpm lint`
|
||||
- **Before committing**: Always run tests and linter on both main backend AND examples
|
||||
- **Go module changes**:
|
||||
- **Any** change to `backend/go.mod` or `backend/go.sum` requires `go mod tidy` in `backend/_example/memory_store` in the same commit. That covers dependency bumps, adding or removing a dependency, and changing the `go` directive, not only version updates.
|
||||
- Only `go mod tidy` there, not `go mod vendor`: the example's vendor directory is gitignored (`.gitignore:26`), so its output is never committed, while a stale local copy silently becomes what the example resolves against.
|
||||
- The example module replaces `github.com/umputun/remark42/backend` with `../../`, so it carries the backend's dependencies as indirect entries. Leaving them stale fails the `test examples` CI step with `go: updates to go.mod needed; to update it: go mod tidy`.
|
||||
- This applies to Dependabot pull requests too: the bot updates `backend/` only, so its Go module PRs need the example tidied before they can go green.
|
||||
|
||||
|
||||
## Backend Test Determinism
|
||||
|
||||
Backend tests must never depend on how fast the machine is. CI runs them under `-race` with coverage on a shared runner, so any test that assumes an operation finishes within some duration eventually fails on a rerun-and-it-passes basis.
|
||||
|
||||
- **Wait on a condition, never on a duration.** Use `require.Eventually` / `require.EventuallyWithT` to poll for the state the assertion needs, and `require.Never` when the point is that something did *not* happen. A bare `time.Sleep` before an assertion is a defect; sleeping until a deadline you computed, as `waitPastMillisecond` does, is not.
|
||||
- **Polling closures must not touch `*testing.T`.** testify runs them on a separate goroutine, where `t.FailNow` is undefined behaviour. Assert on the `*assert.CollectT` that `EventuallyWithT` hands the closure, so the real error also lands in the failure message.
|
||||
- **Mind the rate limiter when polling over HTTP.** Route groups are capped independently and most of the caps are hard-coded in `rest.go`, out of reach of a test: `/auth/` at 2 req/s and the admin, protected and image routes at 10 req/s. Only the open-route group is settable, via `openRouteLimiter` (100 in `startupT`). Poll with the existing constants rather than a new number, `httpPoll` for anything issuing an HTTP request and `pollInterval` only for in-process or filesystem checks, or the poll manufactures the 429s it then has to interpret.
|
||||
- **When a test needs time to have passed, pin the clock input rather than waiting for it:** `os.Chtimes` for file ages, an explicit `store.Comment.Timestamp` for anything that formats a timestamp.
|
||||
- **Prefer a `testing/synctest` bubble** where the code under test has no real I/O. Inside one the clock is fake, so `time.Sleep` is instant and deterministic. `app/notify`, `app/store/service`, `app/store/image`, `app/store/engine`, `app/providers`, `app/migrator` and `_example/memory_store/accessor` already use it, and most surviving `time.Sleep` calls live in them.
|
||||
- **Helpers fail loudly.** A wait that gives up must call `t.Fatal`/`require` naming what it was waiting for, never return silently and leave the next assertion to fail with something unrelated. Because these packages run `goleak.VerifyTestMain`, a failing helper also exits the test goroutine, so anything that started a server in a goroutine must `defer cancel()` or `defer srv.Shutdown()` right after launching it; otherwise a failed readiness wait is reported as a goroutine leak rather than the failure that caused it.
|
||||
- **Take ports and paths from outside the test.** Ports come from the kernel with `net.Listen("tcp", ":0")`, files from `t.TempDir()`. `go test ./...` runs package binaries concurrently, so a number out of a fixed range or a fixed name under `/tmp` lets two of them collide.
|
||||
- **Close idle connections before shutting a test server down.** Clients built as `http.Client{Timeout: x}` share `http.DefaultTransport`, and `Shutdown` waits on their keep-alive connections until its own deadline expires.
|
||||
- **Keep the test timeout budgets aligned.** `Makefile`, `ci-backend.yml`, `release.yml` and the command above all use `-timeout=300s`; the wait helpers allow 30s per condition, so a shorter per-package budget turns a slow runner into a timeout panic instead of a readable failure.
|
||||
|
||||
`chooseUnusedPort` and the server-start wait helpers are duplicated in `app`, `app/cmd`, `app/rest/api` and `_example/memory_store/server`. Nothing shares them today; keep the copies in step when changing one.
|
||||
|
||||
## Release Procedure
|
||||
|
||||
Remark42 uses two tags for each release:
|
||||
- `vX.Y.Z` - product release tag used by GitHub releases, GoReleaser binary artifacts, and Docker image publishing.
|
||||
- `backend/vX.Y.Z` - nested Go module tag for `github.com/umputun/remark42/backend`.
|
||||
|
||||
Release flow:
|
||||
1. Create the GitHub release for `vX.Y.Z` with title `Version X.Y.Z`. The GitHub release must exist before the `vX.Y.Z` tag reaches the remote; `gh release create vX.Y.Z` satisfies this because it creates and pushes the tag.
|
||||
2. The `vX.Y.Z` tag triggers GoReleaser, which builds and uploads binary artifacts to the existing release.
|
||||
3. Create and push the matching backend module tag pointing at the same commit:
|
||||
|
||||
```bash
|
||||
git fetch origin --tags
|
||||
git tag backend/vX.Y.Z vX.Y.Z
|
||||
git push origin backend/vX.Y.Z
|
||||
```
|
||||
|
||||
GoReleaser must ignore `backend/*` tags in `.goreleaser.yml` so release notes and current-tag detection use only product tags. Docker image publishing stays separate and is handled by the existing Docker workflow.
|
||||
|
||||
For local artifact runs, install GoReleaser, Go 1.25, Node 24+ and PNPM 10, then use `make release`. The target runs a snapshot/no-publish GoReleaser build, leaves local artifacts and metadata in `dist/`, and cleans generated frontend embed files after GoReleaser exits. Do not run raw `goreleaser release` for local artifacts unless you also run `./scripts/cleanup-release-assets.sh` afterward.
|
||||
|
||||
## Milestones and Issue Labels
|
||||
|
||||
**Milestones** — one `vX.Y.Z` milestone per release. Assign every merged PR, and every issue closed by a code change, to the milestone of the release it shipped in.
|
||||
- Decide which release a PR belongs to by whether its merge commit is **contained in a release tag** — not by comparing dates (a tag can be cut from an earlier commit, or moved). `git fetch --tags`, then `git tag --contains <merge_sha> | grep '^v' | sort -V | head -1` is its release. If no release tag contains it yet, it belongs to the next (unreleased) version's milestone — create it if missing (`gh api repos/umputun/remark42/milestones -f title="vX.Y.Z"`).
|
||||
- An **issue gets a milestone only when it was closed by a code change** (a linked closing PR/commit); take the milestone from that PR/commit (via the commit-in-tag rule). Issues closed as `duplicate`/`invalid`/`wontfix`/answered get no milestone.
|
||||
- Find unassigned: `gh pr list --state merged --search "no:milestone"`, `gh issue list --state closed --search "no:milestone"`. Assign with `gh pr edit N --milestone "vX.Y.Z"` / `gh issue edit N --milestone "vX.Y.Z"`.
|
||||
|
||||
**Issue labels** — classify each issue with a type and an area (add priority when relevant):
|
||||
- Type: `bug`, `enhancement`, `question`, `documentation`, `discussion`
|
||||
- Area: `backend`, `frontend`, `site`, `CI`, `design`, `localization`
|
||||
- Priority: `important`, `minor`, `some day`
|
||||
- Contribution: `help wanted`, `good-first-issue`
|
||||
- Resolution (on close, when applicable): `duplicate`, `invalid`, `wontfix`, `no-action-needed`
|
||||
- PR auto-labels (applied by Dependabot/Actions, not manual PRs): `dependencies`, `go`, `javascript`, `github_actions`
|
||||
|
||||
## Code Style
|
||||
- **Backend**: Formatting with golangci-lint, strict error handling
|
||||
- **Frontend**: TypeScript with ESLint, Stylelint and Prettier
|
||||
- **Imports**: Group stdlib, external packages, then internal packages
|
||||
- **CSS**: All components use CSS Modules (`component.module.css`). Class naming: BEM block = `.root`, elements = camelCase, modifiers = camelCase. Use `clsx` for conditional class composition. `raw-content.css` is the only global CSS file (syntax highlighting utility). Root wrapper keeps bare `.dark`/`.light` theme class — 8+ module CSS files depend on `:global(.dark)` ancestor. `comment_highlighting` uses `:global()` for imperative `classList` usage in root.tsx
|
||||
|
||||
## Key Backend Packages
|
||||
- **Web/API**: `github.com/go-pkgz/routegroup`, `github.com/go-pkgz/rest`
|
||||
- **Auth**: `github.com/go-pkgz/auth/v2`
|
||||
- **Logging**: `github.com/go-pkgz/lgr`
|
||||
- **Testing**: `github.com/stretchr/testify`
|
||||
- **Notifications**: `github.com/go-pkgz/notify`
|
||||
|
||||
## Repository Structure
|
||||
- Backend: Go server using BoltDB for storage
|
||||
- Frontend: Preact/Redux-based UI with iframe embedding
|
||||
- `/web` is served from two sources, in lookup order: the frontend build output
|
||||
(`frontend/apps/remark42/public`, embedded at `backend/app/cmd/web` or read from `--web-root`),
|
||||
then `backend/app/webassets/assets`, embedded in the binary. A plain page or image the bundler
|
||||
does not process belongs in `webassets`; anything needing templating or the widget's CSS/JS goes
|
||||
through webpack. A name present in both is served from the frontend build.
|
||||
+42
-78
@@ -1,114 +1,78 @@
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine AS frontend-deps
|
||||
|
||||
ARG SKIP_FRONTEND_TEST
|
||||
ARG SKIP_FRONTEND_BUILD
|
||||
# the manifest's prepare script installs husky hooks, which needs a git repository the build
|
||||
# context does not have. husky itself skips on CI, and this is the same flag the build stage sets
|
||||
ENV CI=true
|
||||
|
||||
WORKDIR /srv/frontend/apps/remark42/
|
||||
|
||||
COPY ./frontend/apps/remark42/package.json ./frontend/apps/remark42/pnpm-lock.yaml /srv/frontend/apps/remark42/
|
||||
|
||||
RUN \
|
||||
if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
|
||||
apk add --no-cache --update git && \
|
||||
npm i -g pnpm@10.10.0; \
|
||||
fi
|
||||
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.pnpm-store/v3 \
|
||||
if [[ -z "$SKIP_FRONTEND_BUILD" || -z "$SKIP_FRONTEND_TEST" ]]; then \
|
||||
pnpm i; \
|
||||
fi
|
||||
|
||||
|
||||
FROM --platform=$BUILDPLATFORM frontend-deps AS build-frontend
|
||||
|
||||
ARG SKIP_FRONTEND_TEST
|
||||
ARG SKIP_FRONTEND_BUILD
|
||||
ENV CI=true
|
||||
|
||||
WORKDIR /srv/frontend/apps/remark42/
|
||||
|
||||
COPY ./frontend/apps/remark42/ /srv/frontend/apps/remark42/
|
||||
|
||||
RUN \
|
||||
if [ -z "$SKIP_FRONTEND_TEST" ]; then \
|
||||
pnpm lint type-check translation-check test; \
|
||||
else \
|
||||
echo 'Skip frontend test'; \
|
||||
fi
|
||||
|
||||
RUN \
|
||||
if [ -z "$SKIP_FRONTEND_BUILD" ]; then \
|
||||
pnpm build; \
|
||||
else \
|
||||
mkdir /srv/frontend/apps/remark42/public; \
|
||||
echo 'Skip frontend build'; \
|
||||
fi
|
||||
|
||||
FROM umputun/baseimage:buildgo-v1.17.0 AS build-backend
|
||||
FROM umputun/baseimage:buildgo-latest as build-backend
|
||||
|
||||
ARG CI
|
||||
ARG GITHUB_REF
|
||||
ARG GITHUB_SHA
|
||||
ARG GIT_BRANCH
|
||||
ARG DRONE
|
||||
ARG DRONE_TAG
|
||||
ARG DRONE_COMMIT
|
||||
ARG DRONE_BRANCH
|
||||
ARG DRONE_PULL_REQUEST
|
||||
|
||||
ARG SKIP_BACKEND_TEST
|
||||
ARG BACKEND_TEST_TIMEOUT
|
||||
|
||||
# install gcc in order to be able to go test package with -race
|
||||
RUN apk --no-cache add gcc libc-dev
|
||||
|
||||
ADD backend /build/backend
|
||||
# to embed the frontend files statically into Remark42 binary
|
||||
COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /build/backend/app/cmd/web/
|
||||
ADD .git/ /build/backend/.git/
|
||||
WORKDIR /build/backend
|
||||
|
||||
RUN echo go version: `go version`
|
||||
ENV GOFLAGS="-mod=vendor"
|
||||
|
||||
# run tests
|
||||
RUN \
|
||||
cd app && \
|
||||
if [ -z "$SKIP_BACKEND_TEST" ] ; then \
|
||||
CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
|
||||
cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov && \
|
||||
go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
|
||||
cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
|
||||
golangci-lint run --config ../.golangci.yml ./... ; \
|
||||
else \
|
||||
echo "skip backend tests and linter" \
|
||||
; fi
|
||||
else echo "skip backend tests and linter" ; fi
|
||||
|
||||
# if DRONE presented use DRONE_* git env to make version
|
||||
RUN \
|
||||
version="$(/script/version.sh)" && \
|
||||
if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
|
||||
else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
|
||||
echo "version=$version" && \
|
||||
go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
|
||||
|
||||
FROM umputun/baseimage:app-v1.17.0
|
||||
FROM node:10.11-alpine as build-frontend-deps
|
||||
|
||||
ARG GITHUB_SHA
|
||||
ARG CI
|
||||
ENV HUSKY_SKIP_INSTALL=true
|
||||
|
||||
LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
|
||||
org.opencontainers.image.description="Remark42 comment engine" \
|
||||
org.opencontainers.image.documentation="https://remark42.com/docs/getting-started/" \
|
||||
org.opencontainers.image.licenses="MIT" \
|
||||
org.opencontainers.image.source="https://github.com/umputun/remark42" \
|
||||
org.opencontainers.image.title="Remark42" \
|
||||
org.opencontainers.image.url="https://remark42.com/" \
|
||||
org.opencontainers.image.revision="${GITHUB_SHA}"
|
||||
RUN apk add --no-cache --update git
|
||||
ADD frontend/package.json /srv/frontend/package.json
|
||||
ADD frontend/package-lock.json /srv/frontend/package-lock.json
|
||||
RUN cd /srv/frontend && CI=true npm ci --loglevel warn
|
||||
|
||||
FROM node:10.11-alpine as build-frontend
|
||||
|
||||
ARG CI
|
||||
ARG SKIP_FRONTEND_TEST
|
||||
ARG NODE_ENV=production
|
||||
|
||||
COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
|
||||
ADD frontend /srv/frontend
|
||||
RUN cd /srv/frontend && \
|
||||
if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
|
||||
else echo "skip frontend tests and lint" ; npm run build ; fi && \
|
||||
rm -rf ./node_modules
|
||||
|
||||
FROM umputun/baseimage:app
|
||||
|
||||
WORKDIR /srv
|
||||
|
||||
COPY docker-init.sh /srv/init.sh
|
||||
ADD docker-init.sh /entrypoint.sh
|
||||
ADD backend/scripts/backup.sh /usr/local/bin/backup
|
||||
ADD backend/scripts/restore.sh /usr/local/bin/restore
|
||||
ADD backend/scripts/import.sh /usr/local/bin/import
|
||||
RUN chmod +x /srv/init.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
|
||||
RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
|
||||
|
||||
COPY --from=build-backend /build/backend/remark42 /srv/remark42
|
||||
COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /srv/web/
|
||||
COPY --from=build-frontend /srv/frontend/public/ /srv/web
|
||||
RUN chown -R app:app /srv
|
||||
RUN ln -s /srv/remark42 /usr/bin/remark42
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
|
||||
|
||||
COPY docker-init.sh /srv/init.sh
|
||||
RUN chmod +x /srv/init.sh
|
||||
CMD ["/srv/remark42", "server"]
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
FROM node:10.11-alpine as build-frontend-deps
|
||||
|
||||
ARG CI
|
||||
ARG DRONE
|
||||
ARG DRONE_TAG
|
||||
ARG DRONE_COMMIT
|
||||
ARG DRONE_BRANCH
|
||||
|
||||
ENV SKIP_FRONTEND_TEST=true
|
||||
|
||||
RUN apk add --no-cache --update git
|
||||
ADD frontend/package.json /srv/frontend/package.json
|
||||
ADD frontend/package-lock.json /srv/frontend/package-lock.json
|
||||
RUN cd /srv/frontend && CI=true npm ci
|
||||
|
||||
FROM node:10.11-alpine as build-frontend
|
||||
|
||||
ARG CI
|
||||
ARG NODE_ENV=production
|
||||
ENV SKIP_FRONTEND_TEST=true
|
||||
ENV HUSKY_SKIP_INSTALL=true
|
||||
|
||||
COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
|
||||
ADD frontend /srv/frontend
|
||||
RUN cd /srv/frontend && \
|
||||
npm run build && \
|
||||
rm -rf ./node_modules
|
||||
|
||||
FROM umputun/baseimage:buildgo-latest as build-backend
|
||||
|
||||
ARG GITHUB_TOKEN
|
||||
ENV SKIP_BACKEND_TEST=true
|
||||
|
||||
RUN go get github.com/rakyll/statik
|
||||
|
||||
WORKDIR /build/backend
|
||||
ADD backend /build/backend
|
||||
ADD README.md /build/
|
||||
ADD LICENSE /build/
|
||||
|
||||
ADD .git/ /build/backend/.git/
|
||||
|
||||
COPY --from=build-frontend /srv/frontend/public/ web
|
||||
|
||||
RUN \
|
||||
export WEB_ROOT=/build/backend/web && \
|
||||
sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
|
||||
statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
|
||||
ls -la /build/backend/app/rest/api/statik.go && \
|
||||
ls -la /build/backend/web/
|
||||
|
||||
# if DRONE presented use DRONE_* git env to make version
|
||||
RUN \
|
||||
if [ -z "$DRONE" ] ; then \
|
||||
echo "runs outside of drone" && version=$(/script/git-rev.sh); \
|
||||
else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
|
||||
echo "version=$version" && \
|
||||
export GOFLAGS="-mod=vendor" && \
|
||||
GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
|
||||
GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
|
||||
|
||||
RUN \
|
||||
if [ -z "$DRONE_TAG" ] ; then \
|
||||
echo "runs outside of drone" && tag=""; \
|
||||
else tag=_${DRONE_TAG}; fi && \
|
||||
apk add --no-cache --update zip && \
|
||||
cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
|
||||
tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
|
||||
tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
|
||||
tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
|
||||
tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
|
||||
tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
|
||||
tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
|
||||
zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
|
||||
|
||||
# upload to github
|
||||
#RUN \
|
||||
# if [ -z "$DRONE_TAG" ] ; then \
|
||||
# echo "skip upload to github" ; \
|
||||
# else \
|
||||
# curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
|
||||
# -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
|
||||
# "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
|
||||
# curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
|
||||
# -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
|
||||
# "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
|
||||
# curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
|
||||
# -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
|
||||
# "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
|
||||
# curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
|
||||
# -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
|
||||
# "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
|
||||
# curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
|
||||
# -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
|
||||
# "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
|
||||
|
||||
FROM alpine
|
||||
COPY --from=build-backend /build/backend/remark42.* /artifacts/
|
||||
RUN ls -la /artifacts/*
|
||||
CMD ["sleep", "100"]
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2021 Umputun
|
||||
Copyright (c) 2020 Umputun
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,59 +1,40 @@
|
||||
OS=linux
|
||||
ARCH=amd64
|
||||
GITHUB_REF=$(shell git rev-parse --symbolic-full-name HEAD)
|
||||
GITHUB_SHA=$(shell git rev-parse --short HEAD)
|
||||
CLEANUP_RELEASE_ASSETS=$(CURDIR)/scripts/cleanup-release-assets.sh
|
||||
|
||||
bin:
|
||||
@set -e; \
|
||||
./scripts/prepare-release-assets.sh; \
|
||||
trap '$(CLEANUP_RELEASE_ASSETS)' EXIT; \
|
||||
cd backend && CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o ../remark42 -ldflags "-X main.revision=$(GITHUB_REF)-$(GITHUB_SHA) -s -w" ./app
|
||||
docker build -f Dockerfile.artifacts -t remark42.bin .
|
||||
- @docker rm -f remark42.bin 2>/dev/null || exit 0
|
||||
docker run -d --name=remark42.bin remark42.bin
|
||||
docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
|
||||
docker rm -f remark42.bin
|
||||
|
||||
docker:
|
||||
DOCKER_BUILDKIT=1 docker build -t umputun/remark42 -t ghcr.io/umputun/remark42 --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) \
|
||||
--build-arg CI=true --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
|
||||
docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
|
||||
|
||||
dockerx:
|
||||
docker buildx build --build-arg GITHUB_REF=$(GITHUB_REF) --build-arg GITHUB_SHA=$(GITHUB_SHA) --build-arg CI=true \
|
||||
--build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
|
||||
--progress=plain --platform linux/amd64,linux/arm64 \
|
||||
-t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
|
||||
|
||||
release:
|
||||
@set -e; \
|
||||
trap '$(CLEANUP_RELEASE_ASSETS)' EXIT; \
|
||||
goreleaser release --snapshot --clean --skip=publish
|
||||
deploy:
|
||||
docker build -f Dockerfile.artifacts -t remark42.bin .
|
||||
- @docker rm -f remark42.bin 2>/dev/null || exit 0
|
||||
- @mkdir -p bin
|
||||
docker run -d --name=remark42.bin remark42.bin
|
||||
docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
|
||||
docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
|
||||
docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
|
||||
docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
|
||||
docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
|
||||
docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
|
||||
docker rm -f remark42.bin
|
||||
|
||||
race_test:
|
||||
cd backend/app && go test -race -timeout=300s -count 1 ./...
|
||||
cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
|
||||
|
||||
backend:
|
||||
docker compose -f compose-dev-backend.yml build
|
||||
docker-compose -f compose-dev-backend.yml build
|
||||
|
||||
frontend:
|
||||
docker compose -f compose-dev-frontend.yml build
|
||||
docker-compose -f compose-dev-frontend.yml build
|
||||
|
||||
rundev:
|
||||
SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true GITHUB_REF=$(GITHUB_REF) GITHUB_SHA=$(GITHUB_SHA) CI=true \
|
||||
docker compose -f compose-private.yml build
|
||||
docker compose -f compose-private.yml up
|
||||
SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
|
||||
docker-compose -f compose-private.yml up
|
||||
|
||||
# stamped the same way the suite stamps a stack it starts itself, so one brought up here is
|
||||
# accepted instead of rejected as belonging to another checkout
|
||||
e2e-up:
|
||||
./e2e/tls/generate.sh
|
||||
E2E_STAMP=$$(./e2e/stamp.sh) docker compose -f compose-e2e-test.yml up -d --build --quiet-pull --wait
|
||||
|
||||
e2e-down:
|
||||
docker compose -f compose-e2e-test.yml down -v
|
||||
|
||||
# the suite brings the stack up itself when it finds none, so e2e-up is only worth running
|
||||
# to keep the containers between invocations
|
||||
e2e:
|
||||
cd e2e && go test -tags=e2e -count 1 -timeout 20m ./...
|
||||
|
||||
e2e-ui:
|
||||
cd e2e && E2E_HEADLESS=false E2E_KEEP=1 go test -tags=e2e -count 1 -v -timeout 20m ./...
|
||||
|
||||
.PHONY: bin docker dockerx release race_test backend frontend rundev e2e e2e-up e2e-down e2e-ui
|
||||
.PHONY: bin backend
|
||||
@@ -1,8 +1,10 @@
|
||||
# Remark42 [](https://github.com/umputun/remark42/actions) [](https://hub.docker.com/r/umputun/remark42) [](https://goreportcard.com/report/github.com/umputun/remark42) [](https://coveralls.io/github/umputun/remark42?branch=master) [](https://app.codecov.io/gh/umputun/remark42)
|
||||
# remark42 [](https://github.com/umputun/remark/actions) [](https://goreportcard.com/report/github.com/umputun/remark) [](https://coveralls.io/github/umputun/remark?branch=master)
|
||||
|
||||
Remark42 is a self-hosted, lightweight and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles, or any other place where readers add comments.
|
||||
|
||||
* Social login via Google, Facebook, Microsoft, GitHub, Apple, Yandex, Patreon, Discord, Telegram and custom OAuth2 providers
|
||||
|
||||
Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
|
||||
|
||||
* Social login via Google, Twitter, Facebook, GitHub and Yandex
|
||||
* Login via email
|
||||
* Optional anonymous access
|
||||
* Multi-level nested comments with both tree and plain presentations
|
||||
@@ -14,36 +16,869 @@ Remark42 is a self-hosted, lightweight and simple (yet functional) comment engin
|
||||
* Images upload with drag-and-drop
|
||||
* Extractor for recent comments, cross-post
|
||||
* RSS for all comments and each post
|
||||
* Telegram, Slack, Webhook and email notifications for Admins (get notified for each new comment)
|
||||
* Email and Telegram notifications for users (get notified when someone responds to your comment)
|
||||
* Export data to JSON with automatic backups
|
||||
* Telegram and email notifications
|
||||
* Export data to json with automatic backups
|
||||
* No external databases, everything embedded in a single data file
|
||||
* Fully dockerized and can be deployed in a single command
|
||||
* Self-contained executable can be deployed directly to Linux, Windows and macOS
|
||||
* Self-contained executable can be deployed directly to Linux, Windows and MacOS
|
||||
* Clean, lightweight and customizable UI with white and dark themes
|
||||
* Multi-site mode from a single instance
|
||||
* Integration with automatic SSL (direct and via [nginx-le](https://github.com/nginx-le/nginx-le))
|
||||
* [Privacy focused](https://remark42.com/#privacy)
|
||||
* Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
|
||||
* [Privacy focused](#privacy)
|
||||
|
||||
[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
|
||||
|
||||
<details><summary>Screenshots</summary>
|
||||
|
||||
Comments example:
|
||||
|
||||
|
||||
|
||||
For admin screenshots see [Admin UI documentation](https://remark42.com/docs/manuals/admin-interface/)
|
||||
For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
|
||||
</details>
|
||||
|
||||
All remark42 documentation is available [by the link](https://remark42.com/docs/getting-started/installation/).
|
||||
|
||||
## Contribution
|
||||
#
|
||||
|
||||
In order to start and work on the project locally in development mode check our contribution documentation for [backend](https://remark42.com/docs/contributing/backend/) and [frontend](https://remark42.com/docs/contributing/frontend/).
|
||||
- [Install](#install)
|
||||
- [Backend](#backend)
|
||||
- [With Docker](#with-docker)
|
||||
- [Without Docker](#without-docker)
|
||||
- [Parameters](#parameters)
|
||||
- [Required parameters](#required-parameters)
|
||||
- [Quick installation test](#quick-installation-test)
|
||||
- [Register oauth2 providers](#register-oauth2-providers)
|
||||
- [Google Auth Provider](#google-auth-provider)
|
||||
- [GitHub Auth Provider](#github-auth-provider)
|
||||
- [Facebook Auth Provider](#facebook-auth-provider)
|
||||
- [Twitter Auth Provider](#twitter-auth-provider)
|
||||
- [Yandex Auth Provider](#yandex-auth-provider)
|
||||
- [Initial import from Disqus](#initial-import-from-disqus)
|
||||
- [Initial import from WordPress](#initial-import-from-wordpress)
|
||||
- [Backup and restore](#backup-and-restore)
|
||||
- [Automatic backups](#automatic-backups)
|
||||
- [Manual backup](#manual-backup)
|
||||
- [Restore from backup](#restore-from-backup)
|
||||
- [Backup format](#backup-format)
|
||||
- [Admin users](#admin-users)
|
||||
- [Setup on your website](#setup-on-your-website)
|
||||
- [Comments](#comments)
|
||||
- [Last comments](#last-comments)
|
||||
- [Counter](#counter)
|
||||
- [Build from the source](#build-from-the-source)
|
||||
- [Development](#development)
|
||||
- [Backend development](#backend-development)
|
||||
- [Frontend development](#frontend-development)
|
||||
- [Build](#build)
|
||||
- [Devserver](#devserver)
|
||||
- [API](#api)
|
||||
- [Authorization](#authorization)
|
||||
- [Commenting](#commenting)
|
||||
- [RSS feeds](#rss-feeds)
|
||||
- [Admin](#admin)
|
||||
- [Privacy](#privacy)
|
||||
- [Technical details](#technical-details)
|
||||
|
||||
If you are interested in adding a new localization please check [these docs](https://remark42.com/docs/contributing/translations/).
|
||||
|
||||
## Related projects
|
||||
## Install
|
||||
|
||||
* [A Helm chart for Remark42 on Kubernetes](https://github.com/groundhog2k/helm-charts/tree/master/charts/remark42)
|
||||
* [django-remark42](https://github.com/andrewp-as-is/django-remark42.py)
|
||||
### Backend
|
||||
|
||||
#### With Docker
|
||||
|
||||
_this is the recommended way to run remark42_
|
||||
|
||||
* copy provided `docker-compose.yml` and customize for your needs
|
||||
* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
|
||||
* pull prepared images from the DockerHub and start - `docker-compose pull && docker-compose up -d`
|
||||
* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
|
||||
|
||||
#### Without Docker
|
||||
|
||||
* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
|
||||
* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
|
||||
* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
|
||||
* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Command line | Environment | Default | Description |
|
||||
| ----------------------- | ----------------------- | ------------------------ | ----------------------------------------------- |
|
||||
| url | REMARK_URL | | url to remark42 server, _required_ |
|
||||
| secret | SECRET | | secret key, _required_ |
|
||||
| site | SITE | `remark` | site name(s), _multi_ |
|
||||
| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `rpc` |
|
||||
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
|
||||
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
|
||||
| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ |
|
||||
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email |
|
||||
| backup | BACKUP_PATH | `./var/backup` | backups location |
|
||||
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
|
||||
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
|
||||
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
|
||||
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
|
||||
| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, `bolt`, or `uri` |
|
||||
| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store |
|
||||
| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store |
|
||||
| avatar.uri | AVATAR_URI | `./var/avatars` | avatar store uri |
|
||||
| avatar.rsz-lmt | AVATAR_RSZ_LMT | `0` (disabled) | max image size for resizing avatars on save |
|
||||
| image.type | IMAGE_TYPE | `fs` | type of image storage, `fs`, `bolt` |
|
||||
| image.max-size | IMAGE_MAX_SIZE | `5000000` | max size of image file |
|
||||
| image.fs.path | IMAGE_FS_PATH | `./var/pictures` | permanent location of images |
|
||||
| image.fs.staging | IMAGE_FS_STAGING | `./var/pictures.staging` | staging location of images |
|
||||
| image.fs.partitions | IMAGE_FS_PARTITIONS | `100` | number of image partitions |
|
||||
| image.bolt.file | IMAGE_BOLT_FILE | `/var/pictures.db` | images bolt file location |
|
||||
| image.resize-width | IMAGE_RESIZE_WIDTH | `2400` | width of resized image |
|
||||
| image.resize-height | IMAGE_RESIZE_HEIGHT | `900` | height of resized image |
|
||||
| auth.ttl.jwt | AUTH_TTL_JWT | `5m` | jwt TTL |
|
||||
| auth.ttl.cookie | AUTH_TTL_COOKIE | `200h` | cookie TTL |
|
||||
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
|
||||
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
|
||||
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
|
||||
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
|
||||
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
|
||||
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
|
||||
| auth.twitter.cid | AUTH_TWITTER_CID | | Twitter Consumer API Key |
|
||||
| auth.twitter.csec | AUTH_TWITTER_CSEC | | Twitter Consumer API Secret key |
|
||||
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
|
||||
| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret |
|
||||
| auth.dev | AUTH_DEV | `false` | local oauth2 server, development mode only |
|
||||
| auth.anon | AUTH_ANON | `false` | enable anonymous login |
|
||||
| auth.email.enable | AUTH_EMAIL_ENABLE | `false` | enable auth via email |
|
||||
| auth.email.from | AUTH_EMAIL_FROM | | email from |
|
||||
| auth.email.subj | AUTH_EMAIL_SUBJ | `remark42 confirmation` | email subject |
|
||||
| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html` | email content type |
|
||||
| auth.email.template | AUTH_EMAIL_TEMPLATE | none (predefined) | custom email message template file |
|
||||
| notify.type | NOTIFY_TYPE | none | type of notification (telegram and/or email) |
|
||||
| notify.queue | NOTIFY_QUEUE | `100` | size of notification queue |
|
||||
| notify.telegram.token | NOTIFY_TELEGRAM_TOKEN | | telegram token |
|
||||
| notify.telegram.chan | NOTIFY_TELEGRAM_CHAN | | telegram channel |
|
||||
| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s` | telegram timeout |
|
||||
| notify.email.fromAddress | NOTIFY_EMAIL_FROM | | from email address |
|
||||
| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject |
|
||||
| notify.email.notify_admin | NOTIFY_EMAIL_ADMIN | `false` | notify admin on new comments via ADMIN_SHARED_EMAIL |
|
||||
| smtp.host | SMTP_HOST | | SMTP host |
|
||||
| smtp.port | SMTP_PORT | | SMTP port |
|
||||
| smtp.username | SMTP_USERNAME | | SMTP user name |
|
||||
| smtp.password | SMTP_PASSWORD | | SMTP password |
|
||||
| smtp.tls | SMTP_TLS | | enable TLS for SMTP |
|
||||
| smtp.timeout | SMTP_TIMEOUT | `10s` | SMTP TCP connection timeout |
|
||||
| ssl.type | SSL_TYPE | none | `none`-http, `static`-https, `auto`-https + le |
|
||||
| ssl.port | SSL_PORT | `8443` | port for https server |
|
||||
| ssl.cert | SSL_CERT | | path to cert.pem file |
|
||||
| ssl.key | SSL_KEY | | path to key.pem file |
|
||||
| ssl.acme-location | SSL_ACME_LOCATION | `./var/acme` | dir where obtained le-certs will be stored |
|
||||
| ssl.acme-email | SSL_ACME_EMAIL | | admin email for receiving notifications from LE |
|
||||
| max-comment | MAX_COMMENT_SIZE | `2048` | comment's size limit |
|
||||
| max-votes | MAX_VOTES | `-1` | votes limit per comment, `-1` - unlimited |
|
||||
| votes-ip | VOTES_IP | `false` | restrict votes from the same ip |
|
||||
| anon-vote | ANON_VOTE | `false` | allow voting for anonymous users, require VOTES_IP to be enabled as well |
|
||||
| votes-ip-time | VOTES_IP_TIME | `5m` | same ip vote restriction time, `0s` - unlimited |
|
||||
| low-score | LOW_SCORE | `-5` | low score threshold |
|
||||
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
|
||||
| positive-score | POSITIVE_SCORE | `false` | restricts comment's score to be only positive |
|
||||
| restricted-words | RESTRICTED_WORDS | | words banned in comments (can use `*`), _multi_ |
|
||||
| edit-time | EDIT_TIME | `5m` | edit window |
|
||||
| read-age | READONLY_AGE | | read-only age of comments, days |
|
||||
| image-proxy.http2https | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images |
|
||||
| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false` | enable caching external images to current image storage |
|
||||
| emoji | EMOJI | `false` | enable emoji support |
|
||||
| simple-view | SIMPLE_VIEW | `false` | minimized UI with basic info only |
|
||||
| port | REMARK_PORT | `8080` | web server port |
|
||||
| web-root | REMARK_WEB_ROOT | `./web` | web server root directory |
|
||||
| update-limit | UPDATE_LIMIT | `0.5` | updates/sec limit |
|
||||
| admin-passwd | ADMIN_PASSWD | none (disabled) | password for `admin` basic auth |
|
||||
| dbg | DEBUG | `false` | debug mode |
|
||||
|
||||
* command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
|
||||
* _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
|
||||
* _required_ parameters have to be presented in the environment or provided in command line
|
||||
|
||||
##### Deprecated
|
||||
|
||||
Following list of command-line options is deprecated and will be removed in 2 minor releases or 1 major release (whichever is closer)
|
||||
from the version in which they were deprecated. After remark42 version update, please check startup log once for deprecation warnings to avoid
|
||||
trouble with unrecognized command-line options in the future.
|
||||
|
||||
<details>
|
||||
<summary>deprecated options</summary>
|
||||
|
||||
| Command line | Replacement | Environment | Replacement | Default | Description | Deprecation version |
|
||||
| ------------------ | ------------- | ------------------ | ------------- | ------- | -------------- | ------------------- |
|
||||
| auth.email.host | smtp.host | AUTH_EMAIL_HOST | SMTP_HOST | | smtp host | 1.5.0 |
|
||||
| auth.email.port | smtp.port | AUTH_EMAIL_PORT | SMTP_PORT | | smtp port | 1.5.0 |
|
||||
| auth.email.user | smtp.username | AUTH_EMAIL_USER | SMTP_USERNAME | | smtp user name | 1.5.0 |
|
||||
| auth.email.passwd | smtp.password | AUTH_EMAIL_PASSWD | SMTP_PASSWORD | | smtp password | 1.5.0 |
|
||||
| auth.email.tls | smtp.tls | AUTH_EMAIL_TLS | SMTP_TLS | `false` | enable TLS | 1.5.0 |
|
||||
| auth.email.timeout | smtp.timeout | AUTH_EMAIL_TIMEOUT | SMTP_TIMEOUT | `10s` | smtp timeout | 1.5.0 |
|
||||
| img-proxy | image-proxy.http2https | IMG_PROXY | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images | 1.5.0 |
|
||||
</details>
|
||||
|
||||
##### Required parameters
|
||||
|
||||
Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
|
||||
|
||||
1. `SECRET` - secret key, can be any long and hard-to-guess string.
|
||||
2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.remark42.com`
|
||||
3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
|
||||
|
||||
The minimal `docker-compose.yml` has to include all required parameters:
|
||||
|
||||
```yaml
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
remark42:
|
||||
image: umputun/remark42:latest
|
||||
restart: always
|
||||
container_name: "remark42"
|
||||
environment:
|
||||
- REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server
|
||||
- SITE=YOUR_SITE_ID # site ID, same as used for `site_id`, see "Setup on your website"
|
||||
- SECRET=abcd-123456-xyz-$%^& # secret key
|
||||
- AUTH_GITHUB_CID=12345667890 # oauth2 client ID
|
||||
- AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret
|
||||
volumes:
|
||||
- ./var:/srv/var # persistent volume to store all remark42 data
|
||||
```
|
||||
|
||||
#### Quick installation test
|
||||
|
||||
To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
|
||||
|
||||
#### Register oauth2 providers
|
||||
|
||||
Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
|
||||
|
||||
##### Google Auth Provider
|
||||
|
||||
1. Create a new project: https://console.developers.google.com/project
|
||||
1. Choose the new project from the top right project dropdown (only if another project is selected)
|
||||
1. In the project Dashboard center pane, choose **"API Manager"**
|
||||
1. In the left Nav pane, choose **"Credentials"**
|
||||
1. In the center pane, choose **"OAuth consent screen"** tab. Fill in **"Product name shown to users"** and hit save.
|
||||
1. In the center pane, choose **"Credentials"** tab.
|
||||
* Open the **"New credentials"** drop down
|
||||
* Choose **"OAuth client ID"**
|
||||
* Choose **"Web application"**
|
||||
* Application name is freeform, choose something appropriate
|
||||
* Authorized origins is your domain ex: `https://remark42.mysite.com`
|
||||
* Authorized redirect URIs is the location of oauth2/callback constructed as domain + `/auth/google/callback`, ex: `https://remark42.mysite.com/auth/google/callback`
|
||||
* Choose **"Create"**
|
||||
1. Take note of the **Client ID** and **Client Secret**
|
||||
|
||||
_instructions for google oauth2 setup borrowed from [oauth2_proxy](https://github.com/bitly/oauth2_proxy)_
|
||||
|
||||
##### GitHub Auth Provider
|
||||
|
||||
1. Create a new **"OAuth App"**: https://github.com/settings/developers
|
||||
1. Fill **"Application Name"** and **"Homepage URL"** for your site
|
||||
1. Under **"Authorization callback URL"** enter the correct url constructed as domain + `/auth/github/callback`. ie `https://remark42.mysite.com/auth/github/callback`
|
||||
1. Take note of the **Client ID** and **Client Secret**
|
||||
|
||||
##### Facebook Auth Provider
|
||||
|
||||
1. From https://developers.facebook.com select **"My Apps"** / **"Add a new App"**
|
||||
1. Set **"Display Name"** and **"Contact email"**
|
||||
1. Choose **"Facebook Login"** and then **"Web"**
|
||||
1. Set "Site URL" to your domain, ex: `https://remark42.mysite.com`
|
||||
1. Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
|
||||
1. Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
|
||||
|
||||
##### Twitter Auth Provider
|
||||
|
||||
1. Create a new twitter application https://developer.twitter.com/en/apps
|
||||
1. Fill **App name**, **Description** and **URL** of your site
|
||||
1. In the field **Callback URLs** enter the correct url of your callback handler e.g. domain + `/auth/twitter/callback`
|
||||
1. Under **Key and tokens** take note of the **Consumer API Key** and **Consumer API Secret key**. Those will be used as `AUTH_TWITTER_CID` and
|
||||
`AUTH_TWITTER_CSEC`
|
||||
|
||||
##### Yandex Auth Provider
|
||||
|
||||
1. Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
|
||||
1. Fill **"App name"** for your site
|
||||
1. Under **Platforms** select **"Web services"** and enter **"Callback URI #1"** constructed as domain + `/auth/yandex/callback`. ie `https://remark42.mysite.com/auth/yandex/callback`
|
||||
1. Select **Permissions**. You need following permissions only from the **"Yandex.Passport API"** section:
|
||||
* Access to user avatar
|
||||
* Access to username, first name and surname, gender
|
||||
1. Fill out the rest of fields if needed
|
||||
1. Take note of the **ID** and **Password**
|
||||
|
||||
For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
|
||||
|
||||
##### Anonymous Auth Provider
|
||||
|
||||
Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
|
||||
|
||||
- name should be at least 3 characters long
|
||||
- name has to start from the letter and contains letters, numbers, underscores and spaces only.
|
||||
|
||||
#### Initial import from Disqus
|
||||
|
||||
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
|
||||
2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
|
||||
3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
|
||||
|
||||
#### Initial import from WordPress
|
||||
|
||||
1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
|
||||
2. Move this file to your remark42 host within `./var`
|
||||
3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
|
||||
|
||||
#### Backup and restore
|
||||
|
||||
##### Automatic backups
|
||||
Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./var/backup`). Backups kept up to `${MAX_BACKUP_FILES}` (default 10). Each backup file contains exported and gzipped content, i.e., all comments. At any point, the user can restore such backup and revert all comments to the desirable state. Note: restore procedure cleans the current data store and replaces all comments with comments from the backup file.
|
||||
|
||||
For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
|
||||
|
||||
`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
|
||||
|
||||
##### Manual backup
|
||||
|
||||
In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
|
||||
|
||||
`docker exec -it remark42 backup -s {your site id}`
|
||||
|
||||
##### Restore from backup
|
||||
|
||||
Restore will clean all comments first and then will processed with complete import from a given file.
|
||||
|
||||
`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
|
||||
|
||||
##### Backup format
|
||||
|
||||
Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
|
||||
unmarshaled from `Comment` struct (see below).
|
||||
|
||||
#### Admin users
|
||||
|
||||
Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line.
|
||||
|
||||
```
|
||||
environment:
|
||||
- ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
|
||||
```
|
||||
|
||||
To get user id just login and click on your username or any other user you want to promote to admins.
|
||||
It will expand login info and show full user ID.
|
||||
|
||||
#### Docker parameters
|
||||
|
||||
Two parameters allow customizing Docker container on the system level:
|
||||
|
||||
- `APP_UID` - sets UID to run remark42 application in container (default=1001)
|
||||
- `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
|
||||
|
||||
_see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
|
||||
|
||||
example of `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
remark42:
|
||||
image: umputun/remark42:latest
|
||||
restart: always
|
||||
container_name: "remark42"
|
||||
environment:
|
||||
- APP_UID=2000 # runs remark42 app with non-default UID
|
||||
- TIME_ZONE=GTC # sets container time to UTC
|
||||
|
||||
- REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server
|
||||
- SITE=YOUR_SITE_ID # site ID, same as used for `site_id`, see "Setup on your website"
|
||||
- SECRET=abcd-123456-xyz-$%^& # secret key
|
||||
- AUTH_GITHUB_CID=12345667890 # oauth2 client ID
|
||||
- AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret
|
||||
volumes:
|
||||
- ./var:/srv/var # persistent volume to store all remark42 data
|
||||
```
|
||||
|
||||
|
||||
### Setup on your website
|
||||
|
||||
#### Comments
|
||||
|
||||
It's a main widget which renders list of comments.
|
||||
|
||||
Add this snippet to the bottom of web page:
|
||||
|
||||
```html
|
||||
<script>
|
||||
var remark_config = {
|
||||
host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
|
||||
site_id: 'YOUR_SITE_ID',
|
||||
components: ['embed'], // optional param; which components to load. default to ["embed"]
|
||||
// to load all components define components as ['embed', 'last-comments', 'counter']
|
||||
// available component are:
|
||||
// - 'embed': basic comments widget
|
||||
// - 'last-comments': last comments widget, see `Last Comments` section below
|
||||
// - 'counter': counter widget, see `Counter` section below
|
||||
url: 'PAGE_URL', // optional param; if it isn't defined
|
||||
// `window.location.origin + window.location.pathname` will be used,
|
||||
//
|
||||
// Note that if you use query parameters as significant part of url
|
||||
// (the one that actually changes content on page)
|
||||
// you will have to configure url manually to keep query params, as
|
||||
// `window.location.origin + window.location.pathname` doesn't contain query params and
|
||||
// hash. For example default url for `https://example/com/example-post?id=1#hash`
|
||||
// would be `https://example/com/example-post`.
|
||||
//
|
||||
// The problem with query params is that they often contain useless params added by
|
||||
// various trackers (utm params) and doesn't have defined order, so Remark treats differently
|
||||
// all this examples:
|
||||
// https://example.com/?postid=1&date=2007-02-11
|
||||
// https://example.com/?date=2007-02-11&postid=1
|
||||
// https://example.com/?date=2007-02-11&postid=1&utm_source=google
|
||||
//
|
||||
// If you deal with query parameters make sure you pass only significant part of it
|
||||
// in well defined order
|
||||
max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
|
||||
theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
|
||||
page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
|
||||
locale: 'en' // set up locale and language, if it isn't defined default value ('en') will be used
|
||||
};
|
||||
|
||||
(function(c) {
|
||||
for(var i = 0; i < c.length; i++){
|
||||
var d = document, s = d.createElement('script');
|
||||
s.src = remark_config.host + '/web/' +c[i] +'.js';
|
||||
s.defer = true;
|
||||
(d.head || d.body).appendChild(s);
|
||||
}
|
||||
})(remark_config.components || ['embed']);
|
||||
</script>
|
||||
```
|
||||
|
||||
And then add this node in the place where you want to see Remark42 widget:
|
||||
|
||||
```html
|
||||
<div id="remark42"></div>
|
||||
```
|
||||
|
||||
After that widget will be rendered inside this node.
|
||||
|
||||
##### Themes
|
||||
|
||||
Right now Remark has two themes: light and dark.
|
||||
You can pick one using configuration object,
|
||||
but there is also a possibility to switch between themes in runtime.
|
||||
For this purpose Remark adds to `window` object named `REMARK42`,
|
||||
which contains function `changeTheme`.
|
||||
Just call this function and pass a name of the theme that you want to turn on:
|
||||
|
||||
```js
|
||||
window.REMARK42.changeTheme('light');
|
||||
```
|
||||
|
||||
##### Locales
|
||||
|
||||
Right now Remark is translated to en, ru (partially), de, and fi languages.
|
||||
You can pick one using [configuration object](#setup-on-your-website).
|
||||
|
||||
Do you want translate remark42 to other locale? Please see [this documentation](https://github.com/umputun/remark42/blob/master/docs/translation.md) for details.
|
||||
|
||||
#### Last comments
|
||||
|
||||
It's a widget which renders list of last comments from your site.
|
||||
|
||||
Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
|
||||
|
||||
```html
|
||||
<script>
|
||||
var remark_config = {
|
||||
host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
|
||||
site_id: 'YOUR_SITE_ID',
|
||||
components: ['last-comments']
|
||||
};
|
||||
|
||||
(function(c) {
|
||||
for(var i = 0; i < c.length; i++){
|
||||
var d = document, s = d.createElement('script');
|
||||
s.src = remark_config.host + '/web/' +c[i] +'.js';
|
||||
s.defer = true;
|
||||
(d.head || d.body).appendChild(s);
|
||||
}
|
||||
})(remark_config.components || ['embed']);
|
||||
</script>
|
||||
```
|
||||
|
||||
And then add this node in the place where you want to see last comments widget:
|
||||
|
||||
```html
|
||||
<div class="remark42__last-comments" data-max="50"></div>
|
||||
```
|
||||
|
||||
`data-max` sets the max amount of comments (default: `15`).
|
||||
|
||||
#### Counter
|
||||
|
||||
It's a widget which renders a number of comments for the specified page.
|
||||
|
||||
Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
|
||||
|
||||
```html
|
||||
<script>
|
||||
var remark_config = {
|
||||
host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
|
||||
site_id: 'YOUR_SITE_ID',
|
||||
components: ['counter']
|
||||
};
|
||||
|
||||
(function(c) {
|
||||
for(var i = 0; i < c.length; i++){
|
||||
var d = document, s = d.createElement('script');
|
||||
s.src = remark_config.host + '/web/' +c[i] +'.js';
|
||||
s.defer = true;
|
||||
(d.head || d.body).appendChild(s);
|
||||
}
|
||||
})(remark_config.components || ['embed']);
|
||||
</script>
|
||||
```
|
||||
|
||||
And then add a node like this in the place where you want to see a number of comments:
|
||||
|
||||
```html
|
||||
<span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
|
||||
```
|
||||
|
||||
You can use as many nodes like this as you need to.
|
||||
The script will found all them by the class `remark__counter`,
|
||||
and it will use `data-url` attribute to define the page with comments.
|
||||
|
||||
Also script can use `url` property from `remark_config` object, or `window.location.origin + window.location.pathname` if nothing else is defined.
|
||||
|
||||
## Build from the source
|
||||
|
||||
- to build Docker container - `make docker`. This command will produce container `umputun/remark42`.
|
||||
- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
|
||||
`remark42` file with everything embedded.
|
||||
|
||||
## Development
|
||||
|
||||
You can use fully functional local version to develop and test both frontend & backend. It requires at least 2GB RAM or swap enabled
|
||||
|
||||
To bring it up run:
|
||||
|
||||
```bash
|
||||
# if you mainly work on backend
|
||||
docker-compose -f compose-dev-backend.yml build
|
||||
docker-compose -f compose-dev-backend.yml up
|
||||
# if you mainly work on frontend
|
||||
docker-compose -f compose-dev-frontend.yml build
|
||||
docker-compose -f compose-dev-frontend.yml up
|
||||
```
|
||||
|
||||
It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
|
||||
To access UI demo page go to `127.0.0.1:8080/web`.
|
||||
By default, you would be logged in as `dev_user` which defined as admin.
|
||||
You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
|
||||
|
||||
Backend Docker Compose config by default skips running frontend related tests.
|
||||
Frontend Docker Compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
|
||||
|
||||
### Backend development
|
||||
|
||||
In order to run backend locally (development mode, without Docker) you have to have the latest stable `go` toolchain [installed](https://golang.org/doc/install).
|
||||
|
||||
To run backend - `cd backend; go run app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
|
||||
It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
|
||||
`HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
|
||||
|
||||
### Frontend development
|
||||
|
||||
#### Developer guide
|
||||
|
||||
Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
|
||||
|
||||
#### Build
|
||||
You should have at least 2GB RAM or swap enabled for building
|
||||
|
||||
* install [Node.js 12.11](https://nodejs.org/en/) or higher;
|
||||
* install [NPM 6.13.4](https://www.npmjs.com/package/npm);
|
||||
* run `npm install` inside `./frontend`;
|
||||
* run `npm run build` there;
|
||||
* result files will be saved in `./frontend/public`.
|
||||
|
||||
**Note** Running `npm install` will set up precommit hooks into your git repository.
|
||||
It used to reformat your frontend code using `prettier` and lint with `eslint` and `stylelint` before every commit.
|
||||
|
||||
#### Devserver
|
||||
|
||||
For local development mode with Hot Reloading use `npm start` instead of `npm run build`.
|
||||
In this case `webpack` will serve files using `webpack-dev-server` on `localhost:9000`.
|
||||
By visiting `127.0.0.1:9000/web` you will get a page with main comments widget
|
||||
communicating with demo server backend running on `https://demo.remark42.com`.
|
||||
But you will not be able to login with any oauth providers due to security reasons.
|
||||
|
||||
You can attach to locally running backend by providing `REMARK_URL` environment variable.
|
||||
```sh
|
||||
npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
|
||||
```
|
||||
|
||||
The best way for start local developer enviroment:
|
||||
```sh
|
||||
cp compose-dev-frontend.yml compose-private-frontend.yml
|
||||
docker-compose -f compose-private-frontend.yml up --build
|
||||
cd frontend
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
|
||||
[Redux](https://github.com/zalmoxisus/redux-devtools-extension).
|
||||
|
||||
|
||||
## API
|
||||
|
||||
### Authorization
|
||||
|
||||
* `GET /auth/{provider}/login?from=http://url&site=site_id&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
|
||||
* `GET /auth/logout` - logout
|
||||
|
||||
```go
|
||||
type User struct {
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
Picture string `json:"picture"`
|
||||
Admin bool `json:"admin"`
|
||||
Blocked bool `json:"block"`
|
||||
Verified bool `json:"verified"`
|
||||
}
|
||||
```
|
||||
|
||||
_currently supported providers are `google`, `facebook`, `github` and `yandex`_
|
||||
|
||||
### Commenting
|
||||
|
||||
* `POST /api/v1/comment` - add a comment. _auth required_
|
||||
|
||||
```go
|
||||
type Comment struct {
|
||||
ID string `json:"id"` // comment ID, read only
|
||||
ParentID string `json:"pid"` // parent ID
|
||||
Text string `json:"text"` // comment text, after md processing
|
||||
Orig string `json:"orig"` // original comment text
|
||||
User User `json:"user"` // user info, read only
|
||||
Locator Locator `json:"locator"` // post locator
|
||||
Score int `json:"score"` // comment score, read only
|
||||
Vote int `json:"vote"` // vote for the current user, -1/1/0.
|
||||
Controversy float64 `json:"controversy,omitempty"` // comment controversy, read only
|
||||
Timestamp time.Time `json:"time"` // time stamp, read only
|
||||
Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
|
||||
Pin bool `json:"pin"` // pinned status, read only
|
||||
Delete bool `json:"delete"` // delete status, read only
|
||||
PostTitle string `json:"title"` // post title
|
||||
}
|
||||
|
||||
type Locator struct {
|
||||
SiteID string `json:"site"` // site id
|
||||
URL string `json:"url"` // post url
|
||||
}
|
||||
|
||||
type Edit struct {
|
||||
Timestamp time.Time `json:"time" bson:"time"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
```
|
||||
|
||||
* `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
|
||||
|
||||
* `GET /api/v1/find?site=site-id&url=post-url&sort=fld&format=tree|plain` - find all comments for given post
|
||||
|
||||
This is the primary call used by UI to show comments for given post. It can return comments in two formats - `plain` and `tree`.
|
||||
In plain format result will be sorted list of `Comment`. In tree format this is going to be tree-like object with this structure:
|
||||
|
||||
```go
|
||||
type Tree struct {
|
||||
Nodes []Node `json:"comments"`
|
||||
Info store.PostInfo `json:"info,omitempty"`
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
Comment store.Comment `json:"comment"`
|
||||
Replies []Node `json:"replies,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
|
||||
|
||||
* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation. Body is `EditRequest` json
|
||||
|
||||
```go
|
||||
type EditRequest struct {
|
||||
Text string `json:"text"` // updated text
|
||||
Summary string `json:"summary"` // optional, summary of the edit
|
||||
Delete bool `json:"delete"` // delete flag
|
||||
}{}
|
||||
```
|
||||
|
||||
* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
|
||||
* `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
|
||||
* `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
|
||||
```go
|
||||
type response struct {
|
||||
Comments []store.Comment `json:"comments"`
|
||||
Count int `json:"count"`
|
||||
}{}
|
||||
```
|
||||
* `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
|
||||
* `POST /api/v1/count?site=siteID` - get number of comments for posts from post body (list of post IDs)
|
||||
* `GET /api/v1/list?site=site-id&limit=5&skip=2` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
|
||||
```go
|
||||
type PostInfo struct {
|
||||
URL string `json:"url"`
|
||||
Count int `json:"count"`
|
||||
ReadOnly bool `json:"read_only,omitempty"`
|
||||
FirstTS time.Time `json:"first_time,omitempty"`
|
||||
LastTS time.Time `json:"last_time,omitempty"`
|
||||
}
|
||||
```
|
||||
* `GET /api/v1/user` - get user info, _auth required_
|
||||
* `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_
|
||||
* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream _auth required_
|
||||
* `POST /api/v1/deleteme?site=site-id` - request deletion of user data. _auth required_
|
||||
* `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
|
||||
|
||||
```go
|
||||
type Config struct {
|
||||
Version string `json:"version"`
|
||||
EditDuration int `json:"edit_duration"`
|
||||
MaxCommentSize int `json:"max_comment_size"`
|
||||
Admins []string `json:"admins"`
|
||||
AdminEmail string `json:"admin_email"`
|
||||
Auth []string `json:"auth_providers"`
|
||||
LowScore int `json:"low_score"`
|
||||
CriticalScore int `json:"critical_score"`
|
||||
PositiveScore bool `json:"positive_score"`
|
||||
ReadOnlyAge int `json:"readonly_age"`
|
||||
MaxImageSize int `json:"max_image_size"`
|
||||
EmojiEnabled bool `json:"emoji_enabled"`
|
||||
}
|
||||
```
|
||||
|
||||
* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
|
||||
|
||||
### Streaming API
|
||||
|
||||
Streaming API provide server-sent events for post updates as well as site update
|
||||
|
||||
* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
|
||||
* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
|
||||
|
||||
<details><summary>response example</summary>
|
||||
|
||||
```
|
||||
data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
|
||||
|
||||
event: info
|
||||
data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### RSS feeds
|
||||
|
||||
* `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
|
||||
* `GET /api/v1/rss/site?site=site-id` - rss feed for given site
|
||||
* `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
|
||||
|
||||
### Images management
|
||||
|
||||
* `GET /api/v1/picture/{user}/{id}` - load stored image
|
||||
* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
|
||||
|
||||
_returned id should be appended to load image url on caller side_
|
||||
|
||||
### Email subscription
|
||||
|
||||
* `GET /api/v1/email?site=site-id` - get user's email, _auth required_
|
||||
* `POST /api/v1/email/subscribe?site=site-id&address=user@example.org` - makes confirmation token and sends it to user over email, _auth required_
|
||||
|
||||
Trying to subscribe same email second time will return response code `409 Conflict` and explaining error message.
|
||||
* `POST /api/v1/email/confirm?site=site-id&tkn=token` - uses provided token parameter to set email for the user, _auth required_
|
||||
|
||||
Setting email subscribe user for all first-level replies to his messages.
|
||||
* `DELETE /api/v1/email?site=siteID` - removes user's email, _auth required_
|
||||
|
||||
### Admin
|
||||
|
||||
* `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
|
||||
* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1&ttl=7d` - block or unblock user with optional ttl (default=permanent)
|
||||
* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids
|
||||
```go
|
||||
type BlockedUser struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Until time.Time `json:"time"`
|
||||
}
|
||||
```
|
||||
* `GET /api/v1/admin/export?site=site-id&mode=[stream|file]` - export all comments to json stream or gz file.
|
||||
* `POST /api/v1/admin/import?site=site-id` - import comments from the backup, uses post body.
|
||||
* `POST /api/v1/admin/import/form?site=site-id` - import comments from the backup, user post form.
|
||||
* `POST /api/v1/admin/remap?site=site-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
|
||||
From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
|
||||
export/import chain so make backup first.
|
||||
```
|
||||
http://oldsite.com* https://newsite.com*
|
||||
http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
|
||||
```
|
||||
* `GET /api/v1/admin/wait?site=site-id` - wait for completion for any async migration ops (import or remap).
|
||||
* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
|
||||
* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
|
||||
* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
|
||||
* `PUT /api/v1/admin/readonly?site=site-id&url=post-url&ro=1` - set read-only status
|
||||
* `PUT /api/v1/admin/verify/{userid}?site=site-id&verified=1` - set verified status
|
||||
* `GET /api/v1/admin/deleteme?token=token` - process deleteme user's request
|
||||
|
||||
_all admin calls require auth and admin privilege_
|
||||
|
||||
## Privacy
|
||||
|
||||
* Remark42 is trying to be very sensitive to any private or semi-private information.
|
||||
* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
|
||||
* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
|
||||
* There is no tracking of any sort.
|
||||
* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF.
|
||||
* There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
|
||||
* There are no third-party analytic services involved.
|
||||
* User can request all information remark42 knows about and export to gz file.
|
||||
* Supported complete cleanup of all information related to user's activity.
|
||||
* Cookie lifespan can be restricted to session-only.
|
||||
* All potentially sensitive data stored by remark42 hashed and encrypted.
|
||||
|
||||
## Technical details
|
||||
|
||||
* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `STORE_BOLT_PATH`
|
||||
* Each site stored in a separate boltbd file.
|
||||
* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_FS_PATH` should be transferred. Optionally, boltdb can be used to store avatars as well.
|
||||
* Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
|
||||
* Authentication implemented with [go-pkgz/auth](https://github.com/go-pkgz/auth) stored in a cookie. It uses HttpOnly, secure cookies.
|
||||
* All heavy REST calls cached internally in LRU cache limited by `CACHE_MAX_ITEMS` and `CACHE_MAX_SIZE` with [go-pkgz/rest](https://github.com/go-pkgz/rest)
|
||||
* User's activity throttled globally (up to 1000 simultaneous requests) and limited locally (per user, usually up to 10 req/sec)
|
||||
* Request timeout set to 60sec
|
||||
* Admin authentication (`--admin-password` set) allows to hit remark42 API without social login and with admin privileges. Adds basic-auth for username: `admin`, password: `${ADMIN_PASSWD}`.
|
||||
* User can vote for the comment multiple times but only to change the vote. Double-voting not allowed.
|
||||
* User can edit comments in 5 mins (configurable) window after creation.
|
||||
* User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
|
||||
* All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
|
||||
* Images can be proxied (`IMAGE_PROXY_HTTP2HTTPS=true`) to prevent mixed http/https.
|
||||
* All images can be proxied and saved (`IMAGE_PROXY_CACHE_EXTERNAL=true`) instead of serving from original location. Beware, images which are posted with this parameter enabled will be served from proxy even after it will be disabled.
|
||||
* Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
We release patches for security vulnerabilities.
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| current | :white_check_mark: |
|
||||
| >=1.6.x | :white_check_mark: |
|
||||
| <1.5.x | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report (suspected) security vulnerabilities either by using GitHub's [private vulnerability reporting](https://github.com/umputun/remark42/security/advisories/new) (click the "Report a vulnerability" button on the [Security tab](https://github.com/umputun/remark42/security)) or by emailing umputun@gmail.com. You will receive a response within 48 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.
|
||||
+66
-65
@@ -1,70 +1,71 @@
|
||||
version: "2"
|
||||
run:
|
||||
output:
|
||||
format: tab
|
||||
skip-dirs:
|
||||
- vendor
|
||||
|
||||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: true
|
||||
golint:
|
||||
min-confidence: 0.1
|
||||
maligned:
|
||||
suggest-new: true
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
misspell:
|
||||
locale: US
|
||||
lll:
|
||||
line-length: 140
|
||||
gocritic:
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- bodyclose
|
||||
- copyloopvar
|
||||
- dupl
|
||||
- gochecknoinits
|
||||
- gocritic
|
||||
- gocyclo
|
||||
- gosec
|
||||
- megacheck
|
||||
- golint
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
- nakedret
|
||||
- prealloc
|
||||
- revive
|
||||
- staticcheck
|
||||
- unconvert
|
||||
- megacheck
|
||||
- structcheck
|
||||
- gas
|
||||
- gocyclo
|
||||
- dupl
|
||||
- misspell
|
||||
- unparam
|
||||
- unused
|
||||
settings:
|
||||
gosec:
|
||||
excludes:
|
||||
- G117 # false positive: struct field name matches "secret" pattern
|
||||
gocritic:
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
- hugeParam
|
||||
- rangeValCopy
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
govet:
|
||||
enable:
|
||||
- shadow
|
||||
misspell:
|
||||
locale: US
|
||||
exclusions:
|
||||
generated: lax
|
||||
rules:
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: at least one file in a package should have a package comment
|
||||
- linters:
|
||||
- revive
|
||||
text: 'package-comments: should have a package comment'
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid meaningless package names'
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid package names that conflict with Go standard library package names'
|
||||
- linters:
|
||||
- dupl
|
||||
- gosec
|
||||
path: _test\.go
|
||||
paths:
|
||||
- vendor
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
- varcheck
|
||||
- deadcode
|
||||
- typecheck
|
||||
- ineffassign
|
||||
- varcheck
|
||||
- stylecheck
|
||||
- gochecknoinits
|
||||
- scopelint
|
||||
- nakedret
|
||||
- gosimple
|
||||
- prealloc
|
||||
fast: false
|
||||
disable-all: true
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- text: "at least one file in a package should have a package comment"
|
||||
linters:
|
||||
- stylecheck
|
||||
- text: "should have a package comment, unless it's in another file for this package"
|
||||
linters:
|
||||
- golint
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- gosec
|
||||
- dupl
|
||||
exclude-use-default: false
|
||||
|
||||
service:
|
||||
golangci-lint-version: 1.23.x
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../site/content/docs/contributing/backend/index.md
|
||||
@@ -1,22 +1,15 @@
|
||||
FROM umputun/baseimage:buildgo-v1.17.0 AS build-backend
|
||||
FROM umputun/baseimage:buildgo-latest as build-backend
|
||||
|
||||
#ADD . /build/memory_store
|
||||
#WORKDIR /build/memory_store
|
||||
|
||||
ADD backend /build/backend
|
||||
WORKDIR /build/backend/_example/memory_store
|
||||
|
||||
RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
|
||||
|
||||
FROM umputun/baseimage:app-v1.17.0
|
||||
|
||||
ARG GITHUB_SHA
|
||||
|
||||
LABEL org.opencontainers.image.authors="Umputun <umputun@gmail.com>" \
|
||||
org.opencontainers.image.description="Remark42 comment engine example JRPC memory store" \
|
||||
org.opencontainers.image.documentation="https://github.com/umputun/remark42/tree/master/backend/_example/memory_store" \
|
||||
org.opencontainers.image.licenses="MIT" \
|
||||
org.opencontainers.image.source="https://github.com/umputun/remark42" \
|
||||
org.opencontainers.image.title="Remark42 JRPC example memory store" \
|
||||
org.opencontainers.image.url="https://remark42.com/" \
|
||||
org.opencontainers.image.revision="${GITHUB_SHA}"
|
||||
FROM umputun/baseimage:app-latest
|
||||
|
||||
WORKDIR /srv
|
||||
COPY --from=build-backend /build/bin/memory_store /srv/memory_store
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
# sample store implementation
|
||||
# sample store implementation
|
||||
|
||||
`memory_store` illustrates how to make a custom storage plugin for remark42.
|
||||
`memory_store` illustrates how to make a custom storage plugin for remark42.
|
||||
|
||||
In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
|
||||
|
||||
1. `docker compose -f compose-dev-memstore.yml build`
|
||||
1. `docker compose -f compose-dev-memstore.yml up`
|
||||
1. `docker-compose -f compose-dev-memstore.yml build`
|
||||
1. `docker-compose -f compose-dev-memstore.yml up`
|
||||
|
||||
As usual, demo site will run on http://127.0.0.1:8080/web/
|
||||
|
||||
note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. In real-life usage `replace github.com/umputun/remark42/backend => ../../` should not be used.
|
||||
note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
|
||||
In real-life usage `replace github.com/umputun/remark/backend => ../../` should not be used.
|
||||
@@ -7,11 +7,10 @@
|
||||
package accessor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
)
|
||||
|
||||
// MemAdmin implements admin.Store with memory backend
|
||||
@@ -35,9 +34,8 @@ func NewMemAdminStore(key string) *MemAdmin {
|
||||
return &MemAdmin{data: map[string]AdminRec{}, key: key}
|
||||
}
|
||||
|
||||
// Key supposed to execute find by siteID and returns substructure with secret key,
|
||||
// but in this case the shared secret is used for all sites
|
||||
func (m *MemAdmin) Key(_ string) (key string, err error) {
|
||||
// Key executes find by siteID and returns substructure with secret key
|
||||
func (m *MemAdmin) Key() (key string, err error) {
|
||||
return m.key, nil
|
||||
}
|
||||
|
||||
@@ -45,7 +43,7 @@ func (m *MemAdmin) Key(_ string) (key string, err error) {
|
||||
func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
|
||||
resp, ok := m.data[siteID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("site %s not found", siteID)
|
||||
return nil, errors.Errorf("site %s not found", siteID)
|
||||
}
|
||||
log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
|
||||
return resp.IDs, nil
|
||||
@@ -55,7 +53,7 @@ func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
|
||||
func (m *MemAdmin) Email(siteID string) (email string, err error) {
|
||||
resp, ok := m.data[siteID]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("site %s not found", siteID)
|
||||
return "", errors.Errorf("site %s not found", siteID)
|
||||
}
|
||||
|
||||
return resp.Email, nil
|
||||
@@ -65,7 +63,7 @@ func (m *MemAdmin) Email(siteID string) (email string, err error) {
|
||||
func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
|
||||
resp, ok := m.data[siteID]
|
||||
if !ok {
|
||||
return false, fmt.Errorf("site %s not found", siteID)
|
||||
return false, errors.Errorf("site %s not found", siteID)
|
||||
}
|
||||
return resp.Enabled, nil
|
||||
}
|
||||
@@ -74,7 +72,7 @@ func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
|
||||
func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
|
||||
resp, ok := m.data[siteID]
|
||||
if !ok {
|
||||
return fmt.Errorf("site %s not found", siteID)
|
||||
return errors.Errorf("site %s not found", siteID)
|
||||
}
|
||||
if ev == admin.EvCreate {
|
||||
resp.CountCreated++ // not a good idea, just for demo
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
)
|
||||
|
||||
func TestMemAdmin_Get(t *testing.T) {
|
||||
@@ -31,7 +31,7 @@ func TestMemAdmin_Get(t *testing.T) {
|
||||
email, err := ms.Email("site1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "e1", email)
|
||||
key, err := ms.Key("any")
|
||||
key, err := ms.Key()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "secret", key)
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestMemAdmin_Get(t *testing.T) {
|
||||
email, err = ms.Email("site2")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "e2", email)
|
||||
key, err = ms.Key("any")
|
||||
key, err = ms.Key()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "secret", key)
|
||||
|
||||
|
||||
@@ -7,14 +7,15 @@
|
||||
package accessor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
)
|
||||
|
||||
const lastLimit = 1000
|
||||
@@ -24,7 +25,7 @@ type MemData struct {
|
||||
posts map[string][]store.Comment // key is siteID
|
||||
metaUsers map[string]metaUser // key is userID
|
||||
metaPosts map[store.Locator]metaPost // key is post's locator
|
||||
mu sync.RWMutex
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
type metaPost struct {
|
||||
@@ -57,15 +58,15 @@ func NewMemData() *MemData {
|
||||
func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
|
||||
|
||||
if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
|
||||
return "", fmt.Errorf("post %s is read-only", comment.Locator.URL)
|
||||
return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
comments := m.posts[comment.Locator.SiteID]
|
||||
for _, c := range comments { // don't allow duplicated IDs
|
||||
if c.ID == comment.ID {
|
||||
return "", fmt.Errorf("dup key")
|
||||
return "", errors.New("dup key")
|
||||
}
|
||||
}
|
||||
comments = append(comments, comment)
|
||||
@@ -75,8 +76,8 @@ func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
|
||||
|
||||
// Find returns all comments for post and sorts results
|
||||
func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
comments = []store.Comment{}
|
||||
|
||||
@@ -131,22 +132,22 @@ func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err er
|
||||
|
||||
// Get returns comment for locator.URL and commentID string
|
||||
func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
return m.get(req.Locator, req.CommentID)
|
||||
}
|
||||
|
||||
// Update updates comment for locator.URL with mutable part of comment
|
||||
func (m *MemData) Update(comment store.Comment) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return m.updateComment(comment)
|
||||
}
|
||||
|
||||
// Count returns number of comments for post or user
|
||||
func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
switch {
|
||||
case req.Locator.URL != "": // comment's count for post
|
||||
@@ -160,14 +161,14 @@ func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
|
||||
})
|
||||
return len(comments), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid count request %+v", req)
|
||||
return 0, errors.Errorf("invalid count request %+v", req)
|
||||
}
|
||||
}
|
||||
|
||||
// Info get post(s) meta info
|
||||
func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
res = []store.PostInfo{}
|
||||
|
||||
if req.Locator.URL != "" { // post info
|
||||
@@ -175,7 +176,7 @@ func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error)
|
||||
return c.Locator == req.Locator
|
||||
})
|
||||
if len(comments) == 0 {
|
||||
return nil, fmt.Errorf("not found")
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
info := store.PostInfo{
|
||||
URL: req.Locator.URL,
|
||||
@@ -234,13 +235,13 @@ func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("invalid info request %+v", req)
|
||||
return nil, errors.Errorf("invalid info request %+v", req)
|
||||
}
|
||||
|
||||
// Flag sets and gets flag values
|
||||
func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if req.Update == engine.FlagNonSet { // read flag value, no update requested
|
||||
return m.checkFlag(req), nil
|
||||
@@ -251,11 +252,11 @@ func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
|
||||
|
||||
// ListFlags get list of flagged keys, like blocked & verified user
|
||||
// works for full locator (post flags) or with userID
|
||||
func (m *MemData) ListFlags(req engine.FlagRequest) (res []any, err error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
res = []any{}
|
||||
res = []interface{}{}
|
||||
|
||||
switch req.Flag {
|
||||
case engine.Verified:
|
||||
@@ -267,7 +268,7 @@ func (m *MemData) ListFlags(req engine.FlagRequest) (res []any, err error) {
|
||||
return res, nil
|
||||
|
||||
case engine.Blocked:
|
||||
log.Printf("[INFO] metaUsers: %+v", m.metaUsers)
|
||||
log.Printf("%+v", m.metaUsers)
|
||||
for _, u := range m.metaUsers {
|
||||
if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
|
||||
res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
|
||||
@@ -276,7 +277,7 @@ func (m *MemData) ListFlags(req engine.FlagRequest) (res []any, err error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("flag %s not listable", req.Flag)
|
||||
return nil, errors.Errorf("flag %s not listable", req.Flag)
|
||||
}
|
||||
|
||||
// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
|
||||
@@ -284,43 +285,42 @@ func (m *MemData) ListFlags(req engine.FlagRequest) (res []any, err error) {
|
||||
// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
|
||||
func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
|
||||
switch req.Detail {
|
||||
case engine.UserEmail, engine.UserTelegram:
|
||||
case engine.UserEmail:
|
||||
if req.UserID == "" {
|
||||
return nil, fmt.Errorf("userid cannot be empty in request for single detail")
|
||||
return nil, errors.New("userid cannot be empty in request for single detail")
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if req.Update == "" { // read detail value, no update requested
|
||||
return m.getUserDetail(req), nil
|
||||
return m.getUserDetail(req)
|
||||
}
|
||||
|
||||
return m.setUserDetail(req), nil
|
||||
return m.setUserDetail(req)
|
||||
case engine.AllUserDetails:
|
||||
// list of all details returned in case request is a read request
|
||||
// (Update is not set) and does not have UserID or Detail set
|
||||
if req.Update == "" && req.UserID == "" { // read list of all details
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.listDetails(req.Locator), nil
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return m.listDetails(req.Locator)
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported request with userdetail all")
|
||||
return nil, errors.New("unsupported request with userdetail all")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported detail %q", req.Detail)
|
||||
return nil, errors.Errorf("unsupported detail %q", req.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete post(s), user, comment, user details, or everything
|
||||
func (m *MemData) Delete(req engine.DeleteRequest) error {
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
switch {
|
||||
case req.UserDetail != "": // delete user detail
|
||||
m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
|
||||
return nil
|
||||
return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
|
||||
case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
|
||||
return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
|
||||
|
||||
@@ -333,18 +333,17 @@ func (m *MemData) Delete(req engine.DeleteRequest) error {
|
||||
return e
|
||||
}
|
||||
}
|
||||
m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
|
||||
return nil
|
||||
return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
|
||||
|
||||
case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
|
||||
if _, ok := m.posts[req.Locator.SiteID]; !ok {
|
||||
return fmt.Errorf("not found")
|
||||
return errors.New("not found")
|
||||
}
|
||||
m.posts[req.Locator.SiteID] = []store.Comment{}
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid delete request %+v", req)
|
||||
return errors.Errorf("invalid delete request %+v", req)
|
||||
}
|
||||
|
||||
func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
|
||||
@@ -353,7 +352,7 @@ func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteM
|
||||
return c.Locator == loc && c.ID == id
|
||||
})
|
||||
if len(comments) == 0 {
|
||||
return fmt.Errorf("not found")
|
||||
return errors.New("not found")
|
||||
}
|
||||
|
||||
comments[0].SetDeleted(mode)
|
||||
@@ -391,7 +390,10 @@ func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
|
||||
|
||||
func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
|
||||
|
||||
status := req.Update == engine.FlagTrue
|
||||
status := false
|
||||
if req.Update == engine.FlagTrue {
|
||||
status = true
|
||||
}
|
||||
|
||||
switch req.Flag {
|
||||
|
||||
@@ -428,37 +430,32 @@ func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
|
||||
info.ReadOnly = status
|
||||
m.metaPosts[req.Locator] = info
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to set flag %+v: %w", req, err)
|
||||
}
|
||||
return status, nil
|
||||
return status, errors.Wrapf(err, "failed to set flag %+v", req)
|
||||
}
|
||||
|
||||
// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
|
||||
// as an only element of the slice.
|
||||
func (m *MemData) getUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
|
||||
func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
|
||||
if meta, ok := m.metaUsers[req.UserID]; ok {
|
||||
if meta.SiteID != req.Locator.SiteID {
|
||||
return []engine.UserDetailEntry{}
|
||||
return []engine.UserDetailEntry{}, nil
|
||||
}
|
||||
switch req.Detail {
|
||||
case engine.UserEmail:
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}
|
||||
case engine.UserTelegram:
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return []engine.UserDetailEntry{}
|
||||
return []engine.UserDetailEntry{}, nil
|
||||
}
|
||||
|
||||
// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
|
||||
// element of the slice in case of success
|
||||
func (m *MemData) setUserDetail(req engine.UserDetailRequest) []engine.UserDetailEntry {
|
||||
func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
|
||||
var entry metaUser
|
||||
if meta, ok := m.metaUsers[req.UserID]; ok {
|
||||
if meta.SiteID != req.Locator.SiteID {
|
||||
return []engine.UserDetailEntry{}
|
||||
return []engine.UserDetailEntry{}, nil
|
||||
}
|
||||
entry = meta
|
||||
}
|
||||
@@ -475,49 +472,43 @@ func (m *MemData) setUserDetail(req engine.UserDetailRequest) []engine.UserDetai
|
||||
case engine.UserEmail:
|
||||
entry.Details.Email = req.Update
|
||||
m.metaUsers[req.UserID] = entry
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}
|
||||
case engine.UserTelegram:
|
||||
entry.Details.Telegram = req.Update
|
||||
m.metaUsers[req.UserID] = entry
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}
|
||||
return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
|
||||
}
|
||||
|
||||
return []engine.UserDetailEntry{}
|
||||
return []engine.UserDetailEntry{}, nil
|
||||
}
|
||||
|
||||
// listDetails lists all available users details for given siteID
|
||||
func (m *MemData) listDetails(loc store.Locator) []engine.UserDetailEntry {
|
||||
func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
|
||||
var res []engine.UserDetailEntry
|
||||
for _, u := range m.metaUsers {
|
||||
if u.SiteID == loc.SiteID {
|
||||
res = append(res, u.Details)
|
||||
}
|
||||
}
|
||||
return res
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
|
||||
// deletion of the absent entry doesn't produce error.
|
||||
// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
|
||||
func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) {
|
||||
func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
|
||||
var entry metaUser
|
||||
if meta, ok := m.metaUsers[userID]; ok {
|
||||
if meta.SiteID != locator.SiteID {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
entry = meta
|
||||
}
|
||||
|
||||
if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
|
||||
// absent entry means that we should not do anything
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
switch userDetail {
|
||||
case engine.UserEmail:
|
||||
entry.Details.Email = ""
|
||||
case engine.UserTelegram:
|
||||
entry.Details.Telegram = ""
|
||||
case engine.AllUserDetails:
|
||||
entry.Details = engine.UserDetailEntry{UserID: userID}
|
||||
}
|
||||
@@ -528,6 +519,7 @@ func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDet
|
||||
}
|
||||
|
||||
m.metaUsers[userID] = entry
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
|
||||
@@ -535,7 +527,7 @@ func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error
|
||||
return c.Locator == loc && c.ID == commentID
|
||||
})
|
||||
if len(comments) == 0 {
|
||||
return store.Comment{}, fmt.Errorf("not found")
|
||||
return store.Comment{}, errors.New("not found")
|
||||
}
|
||||
return comments[0], nil
|
||||
}
|
||||
@@ -543,21 +535,20 @@ func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error
|
||||
func (m *MemData) updateComment(comment store.Comment) error {
|
||||
comments := m.posts[comment.Locator.SiteID]
|
||||
for i, c := range comments {
|
||||
if c.ID != comment.ID || c.Locator != comment.Locator {
|
||||
continue
|
||||
if c.ID == comment.ID && c.Locator == comment.Locator {
|
||||
c.Text = comment.Text
|
||||
c.Orig = comment.Orig
|
||||
c.Score = comment.Score
|
||||
c.Votes = comment.Votes
|
||||
c.Pin = comment.Pin
|
||||
c.Deleted = comment.Deleted
|
||||
c.User = comment.User
|
||||
comments[i] = c
|
||||
m.posts[comment.Locator.SiteID] = comments
|
||||
return nil
|
||||
}
|
||||
c.Text = comment.Text
|
||||
c.Orig = comment.Orig
|
||||
c.Score = comment.Score
|
||||
c.Votes = comment.Votes
|
||||
c.Pin = comment.Pin
|
||||
c.Deleted = comment.Deleted
|
||||
c.User = comment.User
|
||||
comments[i] = c
|
||||
m.posts[comment.Locator.SiteID] = comments
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("not found")
|
||||
return errors.New("not found")
|
||||
}
|
||||
|
||||
func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
|
||||
|
||||
@@ -10,14 +10,13 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
)
|
||||
|
||||
func TestMemData_CreateAndFind(t *testing.T) {
|
||||
@@ -199,7 +198,7 @@ func TestMemData_FindForUserPagination(t *testing.T) {
|
||||
}
|
||||
|
||||
// write 200 comments
|
||||
for i := range 200 {
|
||||
for i := 0; i < 200; i++ {
|
||||
c.ID = fmt.Sprintf("idd-%d", i)
|
||||
c.Text = fmt.Sprintf("text #%d", i)
|
||||
c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
|
||||
@@ -287,7 +286,7 @@ func TestMemData_CountUser(t *testing.T) {
|
||||
|
||||
func TestMemData_InfoPost(t *testing.T) {
|
||||
b := prepMem(t)
|
||||
ts := func(minute int) time.Time { return time.Date(2017, 12, 20, 15, 18, minute, 0, time.Local).In(time.UTC) }
|
||||
ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
|
||||
|
||||
// add one more for https://radio-t.com/2
|
||||
comment := store.Comment{
|
||||
@@ -485,7 +484,7 @@ func TestMemData_FlagVerified(t *testing.T) {
|
||||
func TestMemData_FlagListVerified(t *testing.T) {
|
||||
|
||||
b := prepMem(t)
|
||||
toIDs := func(inp []any) (res []string) {
|
||||
toIDs := func(inp []interface{}) (res []string) {
|
||||
res = make([]string, len(inp))
|
||||
for i, v := range inp {
|
||||
vv, ok := v.(string)
|
||||
@@ -522,52 +521,51 @@ func TestMemData_FlagListVerified(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMemData_FlagListBlocked(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
b := prepMem(t)
|
||||
setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
|
||||
req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
|
||||
TTL: ttl}
|
||||
_, err := b.Flag(req)
|
||||
return err
|
||||
|
||||
b := prepMem(t)
|
||||
setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
|
||||
req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
|
||||
TTL: ttl}
|
||||
_, err := b.Flag(req)
|
||||
return err
|
||||
}
|
||||
|
||||
toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
|
||||
res = make([]store.BlockedUser, len(inp))
|
||||
for i, v := range inp {
|
||||
vv, ok := v.(store.BlockedUser)
|
||||
require.True(t, ok)
|
||||
res[i] = vv
|
||||
}
|
||||
return res
|
||||
}
|
||||
assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
|
||||
assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
|
||||
assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
|
||||
|
||||
toBlocked := func(inp []any) (res []store.BlockedUser) {
|
||||
res = make([]store.BlockedUser, len(inp))
|
||||
for i, v := range inp {
|
||||
vv, ok := v.(store.BlockedUser)
|
||||
require.True(t, ok)
|
||||
res[i] = vv
|
||||
}
|
||||
return res
|
||||
}
|
||||
assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
|
||||
assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
|
||||
assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
|
||||
vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
|
||||
assert.NoError(t, err)
|
||||
blockedList := toBlocked(vv)
|
||||
var blockedIds = make([]string, len(blockedList))
|
||||
for i, x := range blockedList {
|
||||
blockedIds[i] = x.ID
|
||||
}
|
||||
require.Equal(t, 2, len(blockedList), b.metaUsers)
|
||||
assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
|
||||
t.Logf("%+v", blockedList)
|
||||
|
||||
blockedList := toBlocked(vv)
|
||||
var blockedIDs = make([]string, len(blockedList))
|
||||
for i, x := range blockedList {
|
||||
blockedIDs[i] = x.ID
|
||||
}
|
||||
require.Equal(t, 2, len(blockedList), b.metaUsers)
|
||||
assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIDs)
|
||||
t.Logf("%+v", blockedList)
|
||||
// check block expiration
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
|
||||
assert.NoError(t, err)
|
||||
blockedList = toBlocked(vv)
|
||||
require.Equal(t, 1, len(blockedList))
|
||||
assert.Equal(t, "user1", blockedList[0].ID)
|
||||
|
||||
// check block expiration
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
|
||||
assert.NoError(t, err)
|
||||
blockedList = toBlocked(vv)
|
||||
require.Equal(t, 1, len(blockedList))
|
||||
assert.Equal(t, "user1", blockedList[0].ID)
|
||||
|
||||
vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(vv))
|
||||
})
|
||||
vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(vv))
|
||||
}
|
||||
|
||||
func TestMemData_DeleteComment(t *testing.T) {
|
||||
@@ -626,7 +624,6 @@ func TestMemData_DeleteComment(t *testing.T) {
|
||||
func TestMemData_Close(t *testing.T) {
|
||||
b := prepMem(t)
|
||||
assert.NoError(t, b.Close())
|
||||
assert.NoError(t, b.Close(), "second call should not result in panic or errors")
|
||||
}
|
||||
|
||||
func TestMemData_DeleteHard(t *testing.T) {
|
||||
@@ -661,40 +658,12 @@ func TestMemData_DeleteAll(t *testing.T) {
|
||||
assert.Equal(t, 0, len(comments), "nothing left")
|
||||
}
|
||||
|
||||
func TestMemData_UserDetailAll(t *testing.T) {
|
||||
b := prepMem(t)
|
||||
|
||||
val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, val)
|
||||
}
|
||||
|
||||
func TestMemData_UserDetailErrors(t *testing.T) {
|
||||
b := prepMem(t)
|
||||
|
||||
val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.UserEmail, Update: "value1"})
|
||||
require.EqualError(t, err, "userid cannot be empty in request for single detail")
|
||||
require.Nil(t, val)
|
||||
|
||||
val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails, Update: "value1"})
|
||||
require.EqualError(t, err, "unsupported request with userdetail all")
|
||||
require.Nil(t, val)
|
||||
|
||||
val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: "bad"})
|
||||
require.EqualError(t, err, "unsupported detail \"bad\"")
|
||||
require.Nil(t, val)
|
||||
}
|
||||
|
||||
func TestMemData_DeleteUserDetail(t *testing.T) {
|
||||
var (
|
||||
createEmailUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
|
||||
readEmailUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
|
||||
createTelegramUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram, Update: "value1"}
|
||||
readTelegramUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram}
|
||||
emailSet = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
|
||||
emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
|
||||
telegramSet = []engine.UserDetailEntry{{UserID: "user1", Telegram: "value1"}}
|
||||
telegramUnset = []engine.UserDetailEntry{{UserID: "user1", Telegram: ""}}
|
||||
createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
|
||||
readUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
|
||||
emailSet = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
|
||||
emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
|
||||
)
|
||||
|
||||
b := prepMem(t)
|
||||
@@ -705,21 +674,15 @@ func TestMemData_DeleteUserDetail(t *testing.T) {
|
||||
expected []engine.UserDetailEntry
|
||||
}{
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
|
||||
detailReq: createEmailUser, expected: emailSet},
|
||||
detailReq: createUser, expected: emailSet},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
|
||||
detailReq: readEmailUser, expected: emailSet},
|
||||
detailReq: readUser, expected: emailSet},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
|
||||
detailReq: readEmailUser, expected: emailUnset},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
|
||||
detailReq: createTelegramUser, expected: telegramSet},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserTelegram},
|
||||
detailReq: readTelegramUser, expected: telegramSet},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
|
||||
detailReq: readTelegramUser, expected: telegramUnset},
|
||||
detailReq: readUser, expected: emailUnset},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
|
||||
detailReq: createEmailUser, expected: emailSet},
|
||||
detailReq: createUser, expected: emailSet},
|
||||
{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
|
||||
detailReq: readEmailUser, expected: emailUnset},
|
||||
detailReq: readUser, expected: emailUnset},
|
||||
}
|
||||
|
||||
for i, x := range testData {
|
||||
|
||||
@@ -8,13 +8,13 @@ package accessor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/image"
|
||||
"github.com/umputun/remark/backend/app/store/image"
|
||||
)
|
||||
|
||||
// MemImage implements image.Store with memory backend
|
||||
@@ -22,7 +22,7 @@ type MemImage struct {
|
||||
imagesStaging map[string][]byte
|
||||
images map[string][]byte
|
||||
insertTime map[string]time.Time
|
||||
mu sync.RWMutex
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
// NewMemImageStore makes admin Store in memory.
|
||||
@@ -37,62 +37,40 @@ func NewMemImageStore() *MemImage {
|
||||
|
||||
// Save stores image with passed id to staging
|
||||
func (m *MemImage) Save(id string, img []byte) error {
|
||||
m.mu.Lock()
|
||||
m.Lock()
|
||||
m.imagesStaging[id] = img
|
||||
m.insertTime[id] = time.Now()
|
||||
m.mu.Unlock()
|
||||
m.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetCleanupTimer resets cleanup timer for the image
|
||||
func (m *MemImage) ResetCleanupTimer(id string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, ok := m.insertTime[id]; ok {
|
||||
m.insertTime[id] = time.Now()
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("image %s not found", id)
|
||||
}
|
||||
|
||||
// Load image by ID
|
||||
func (m *MemImage) Load(id string) ([]byte, error) {
|
||||
m.mu.RLock()
|
||||
m.RLock()
|
||||
img, ok := m.images[id]
|
||||
if !ok {
|
||||
img, ok = m.imagesStaging[id]
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
m.RUnlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("image %s not found", id)
|
||||
return nil, errors.Errorf("image %s not found", id)
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// Delete image by ID
|
||||
func (m *MemImage) Delete(id string) error {
|
||||
m.mu.Lock()
|
||||
// delete key from permanent and staging storage
|
||||
delete(m.images, id)
|
||||
delete(m.insertTime, id)
|
||||
delete(m.imagesStaging, id)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Commit moves image from staging to permanent
|
||||
func (m *MemImage) Commit(id string) error {
|
||||
m.mu.RLock()
|
||||
m.RLock()
|
||||
img, ok := m.imagesStaging[id]
|
||||
m.mu.RUnlock()
|
||||
m.RUnlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to commit %s, not found in staging", id)
|
||||
return errors.Errorf("failed to commit %s, not found in staging", id)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.Lock()
|
||||
m.images[id] = img
|
||||
m.mu.Unlock()
|
||||
m.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -101,7 +79,7 @@ func (m *MemImage) Commit(id string) error {
|
||||
func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
|
||||
var idsToRemove []string
|
||||
|
||||
m.mu.RLock()
|
||||
m.RLock()
|
||||
for id, t := range m.insertTime {
|
||||
age := time.Since(t)
|
||||
if age > ttl {
|
||||
@@ -109,27 +87,27 @@ func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
|
||||
idsToRemove = append(idsToRemove, id)
|
||||
}
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
m.RUnlock()
|
||||
|
||||
m.mu.Lock()
|
||||
m.Lock()
|
||||
for _, id := range idsToRemove {
|
||||
delete(m.insertTime, id)
|
||||
delete(m.imagesStaging, id)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
m.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Info returns meta information about storage
|
||||
func (m *MemImage) Info() (image.StoreInfo, error) {
|
||||
var ts time.Time
|
||||
m.mu.RLock()
|
||||
m.RLock()
|
||||
for _, t := range m.insertTime {
|
||||
if ts.IsZero() || t.Before(ts) {
|
||||
ts = t
|
||||
}
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
m.RUnlock()
|
||||
|
||||
return image.StoreInfo{FirstStagingImageTS: ts}, nil
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
// gopher png for test, from https://golang.org/src/image/png/example_test.go
|
||||
const rawGopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
|
||||
const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
|
||||
"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
|
||||
"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
|
||||
"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
|
||||
@@ -38,13 +39,11 @@ const rawGopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62Xe
|
||||
"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
|
||||
"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
|
||||
|
||||
func gopherPNG() io.Reader {
|
||||
return base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawGopher))
|
||||
}
|
||||
func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
|
||||
|
||||
func TestMemImage_LoadAfterSave(t *testing.T) {
|
||||
svc := NewMemImageStore()
|
||||
gopher, err := io.ReadAll(gopherPNG())
|
||||
gopher, err := ioutil.ReadAll(gopherPNG())
|
||||
assert.NoError(t, err)
|
||||
|
||||
img, err := svc.Load("test_id")
|
||||
@@ -59,9 +58,6 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, gopher, img)
|
||||
|
||||
err = svc.ResetCleanupTimer(id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = svc.Commit(id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -73,26 +69,6 @@ func TestMemImage_LoadAfterSave(t *testing.T) {
|
||||
assert.Equal(t, gopher, img)
|
||||
}
|
||||
|
||||
func TestMemImage_LoadAfterDelete(t *testing.T) {
|
||||
svc := NewMemImageStore()
|
||||
gopher, err := io.ReadAll(gopherPNG())
|
||||
assert.NoError(t, err)
|
||||
|
||||
id := "test_img"
|
||||
err = svc.Save(id, gopher)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = svc.Delete(id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
img, err := svc.Load(id)
|
||||
assert.EqualError(t, err, "image test_img not found")
|
||||
assert.Empty(t, img)
|
||||
|
||||
err = svc.ResetCleanupTimer(id)
|
||||
assert.EqualError(t, err, "image test_img not found")
|
||||
}
|
||||
|
||||
func TestMemImage_CommitFail(t *testing.T) {
|
||||
svc := NewMemImageStore()
|
||||
err := svc.Commit("test_id")
|
||||
@@ -107,7 +83,7 @@ func TestMemImage_Cleanup(t *testing.T) {
|
||||
|
||||
func TestMemImage_Info(t *testing.T) {
|
||||
svc := NewMemImageStore()
|
||||
gopher, err := io.ReadAll(gopherPNG())
|
||||
gopher, err := ioutil.ReadAll(gopherPNG())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// get info on empty storage, should be zero
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
|
||||
# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
|
||||
|
||||
version: "2"
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
|
||||
remark42:
|
||||
build:
|
||||
context: ../../..
|
||||
@@ -11,7 +12,7 @@ services:
|
||||
args:
|
||||
- SKIP_BACKEND_TEST=true
|
||||
- SKIP_FRONTEND_TEST=true
|
||||
image: ghcr.io/umputun/remark42:dev
|
||||
image: umputun/remark42:dev
|
||||
container_name: "remark42-dev"
|
||||
hostname: "remark42-dev"
|
||||
restart: always
|
||||
@@ -29,6 +30,7 @@ services:
|
||||
environment:
|
||||
- REMARK_URL=http://127.0.0.1:8080
|
||||
- SECRET=123456
|
||||
- BACKUP_PATH=/srv/var/backup
|
||||
- DEBUG=true
|
||||
- EMOJI=true
|
||||
- AUTH_ANON=true
|
||||
|
||||
@@ -1,34 +1,14 @@
|
||||
module github.com/umputun/remark42/memory_store
|
||||
module github.com/umputun/remark/memory_store
|
||||
|
||||
go 1.25.0
|
||||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/go-pkgz/jrpc v0.4.2
|
||||
github.com/go-pkgz/lgr v0.12.4
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/umputun/remark42/backend v1.1000.0
|
||||
github.com/go-pkgz/jrpc v0.1.0
|
||||
github.com/go-pkgz/lgr v0.7.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/stretchr/testify v1.5.1
|
||||
github.com/umputun/go-flags v1.5.1
|
||||
github.com/umputun/remark/backend v1.5.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Depado/bfchroma/v2 v2.0.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.12.0 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 // indirect
|
||||
github.com/go-pkgz/rest v1.24.0 // indirect
|
||||
github.com/go-pkgz/routegroup v1.6.1 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/image v0.45.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/umputun/remark42/backend v1.1000.0 => ../../
|
||||
replace github.com/umputun/remark/backend => ../../
|
||||
|
||||
@@ -1,52 +1,269 @@
|
||||
github.com/Depado/bfchroma/v2 v2.0.0 h1:IRpN9BPkNwEpR6w1ectIcNWOuhDSLx+8f1pn83fzxx8=
|
||||
github.com/Depado/bfchroma/v2 v2.0.0/go.mod h1:wFwW/Pw8Tnd0irzgO9Zxtxgzp3aPS8qBWlyadxujxmw=
|
||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU=
|
||||
github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ=
|
||||
github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok=
|
||||
github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8=
|
||||
github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI=
|
||||
github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc=
|
||||
github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q=
|
||||
github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
|
||||
github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
|
||||
github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
|
||||
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
|
||||
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
|
||||
github.com/alecthomas/chroma v0.6.0 h1:gcvXlpe0/NoQP3BvneRfgcauLIJDw9VblkoFwZ5XGFs=
|
||||
github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
|
||||
github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
|
||||
github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
|
||||
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
|
||||
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
|
||||
github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
|
||||
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
|
||||
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
|
||||
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
|
||||
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
|
||||
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
|
||||
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
|
||||
github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
|
||||
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
|
||||
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
|
||||
github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
|
||||
github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
|
||||
github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
|
||||
github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
|
||||
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/go-chi/chi v4.1.0+incompatible h1:ETj3cggsVIY2Xao5ExCu6YhEh5MD6JTfcBzS37R260w=
|
||||
github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
|
||||
github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
|
||||
github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
|
||||
github.com/go-pkgz/auth v0.10.1/go.mod h1:wxyQqc0UUP1jT4l6zk1r6XPcVdcgIzW2OiQ8hBEHd64=
|
||||
github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
|
||||
github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
|
||||
github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
|
||||
github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
|
||||
github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
|
||||
github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
|
||||
github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
|
||||
github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
|
||||
github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
|
||||
github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
|
||||
github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
|
||||
github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
|
||||
github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
|
||||
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
|
||||
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
|
||||
github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
|
||||
github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
|
||||
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
|
||||
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
|
||||
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
|
||||
github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
|
||||
github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
|
||||
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
|
||||
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
|
||||
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
|
||||
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
||||
@@ -12,10 +12,10 @@ import (
|
||||
|
||||
"github.com/go-pkgz/jrpc"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/umputun/remark42/memory_store/accessor"
|
||||
"github.com/umputun/remark42/memory_store/server"
|
||||
"github.com/umputun/remark/memory_store/accessor"
|
||||
"github.com/umputun/remark/memory_store/server"
|
||||
)
|
||||
|
||||
// opts with all cli commands and flags
|
||||
@@ -43,14 +43,16 @@ func main() {
|
||||
adminStore := accessor.NewMemAdminStore(opts.Secret)
|
||||
imgStore := accessor.NewMemImageStore()
|
||||
|
||||
rpcServer := jrpc.NewServer(
|
||||
opts.API,
|
||||
jrpc.Auth(opts.AuthUser, opts.AuthPasswd),
|
||||
jrpc.WithSignature("remark42-memory", "umputun", revision),
|
||||
jrpc.WithLogger(log.Default()),
|
||||
)
|
||||
rpcServer := jrpc.Server{
|
||||
API: opts.API,
|
||||
AuthUser: opts.AuthUser,
|
||||
AuthPasswd: opts.AuthPasswd,
|
||||
Version: revision,
|
||||
AppName: "remark42-memory",
|
||||
Logger: log.Default(),
|
||||
}
|
||||
|
||||
srv := server.NewRPC(dataStore, adminStore, imgStore, rpcServer)
|
||||
srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
|
||||
|
||||
admRec := accessor.AdminRec{
|
||||
SiteID: "remark",
|
||||
|
||||
@@ -11,17 +11,12 @@ import (
|
||||
|
||||
"github.com/go-pkgz/jrpc"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
)
|
||||
|
||||
// get admin key
|
||||
func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var siteID string
|
||||
if err := json.Unmarshal(params, &siteID); err != nil {
|
||||
return jrpc.Response{Error: err.Error()}
|
||||
}
|
||||
|
||||
key, err := s.adm.Key(siteID)
|
||||
func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
|
||||
key, err := s.adm.Key()
|
||||
if err != nil {
|
||||
return jrpc.Response{Error: err.Error()}
|
||||
}
|
||||
@@ -73,7 +68,7 @@ func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Respons
|
||||
// onEvent returns nothing, callback to OnEvent
|
||||
func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var siteID string
|
||||
var ps []any
|
||||
var ps []interface{}
|
||||
if err := json.Unmarshal(params, &ps); err != nil {
|
||||
return jrpc.Response{Error: err.Error()}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"github.com/go-pkgz/jrpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
)
|
||||
|
||||
func TestRPC_admKeyHndl(t *testing.T) {
|
||||
@@ -24,7 +24,7 @@ func TestRPC_admKeyHndl(t *testing.T) {
|
||||
api := fmt.Sprintf("http://localhost:%d/test", port)
|
||||
|
||||
ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
|
||||
key, err := ra.Key("any")
|
||||
key, err := ra.Key()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "secret", key)
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
|
||||
"github.com/go-pkgz/jrpc"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
)
|
||||
|
||||
func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
|
||||
@@ -16,8 +16,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
)
|
||||
|
||||
func TestRPC_createHndl(t *testing.T) {
|
||||
@@ -198,62 +198,25 @@ func TestRPC_listFlagsHndl(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "123456", id)
|
||||
|
||||
// verify user
|
||||
verifyFlagReq := engine.FlagRequest{
|
||||
flagReq := engine.FlagRequest{
|
||||
Flag: engine.Verified,
|
||||
UserID: "u1",
|
||||
Locator: store.Locator{
|
||||
SiteID: "test-site",
|
||||
},
|
||||
}
|
||||
flags, err := re.ListFlags(verifyFlagReq)
|
||||
flags, err := re.ListFlags(flagReq)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, flags)
|
||||
assert.Equal(t, []interface{}{}, flags)
|
||||
|
||||
verifyFlagReq.Update = engine.FlagTrue
|
||||
status, err := re.Flag(verifyFlagReq)
|
||||
flagReq.Update = engine.FlagTrue
|
||||
status, err := re.Flag(flagReq)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, true, status)
|
||||
|
||||
flags, err = re.ListFlags(verifyFlagReq)
|
||||
flags, err = re.ListFlags(flagReq)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []any{"u1"}, flags)
|
||||
verifiedUsers := make([]string, 0, len(flags))
|
||||
for _, v := range flags {
|
||||
verifiedUsers = append(verifiedUsers, v.(string))
|
||||
}
|
||||
assert.Equal(t, []string{"u1"}, verifiedUsers)
|
||||
|
||||
// block user
|
||||
blockFlagReq := engine.FlagRequest{
|
||||
Flag: engine.Blocked,
|
||||
UserID: "u1",
|
||||
Locator: store.Locator{
|
||||
SiteID: "test-site",
|
||||
},
|
||||
TTL: time.Hour,
|
||||
}
|
||||
flags, err = re.ListFlags(blockFlagReq)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, flags)
|
||||
|
||||
blockFlagReq.Update = engine.FlagTrue
|
||||
status, err = re.Flag(blockFlagReq)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, true, status)
|
||||
|
||||
flags, err = re.ListFlags(blockFlagReq)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, flags)
|
||||
blockedUsers := make([]store.BlockedUser, 0, len(flags))
|
||||
for _, v := range flags {
|
||||
blockedUsers = append(blockedUsers, v.(store.BlockedUser))
|
||||
}
|
||||
require.Equal(t, 1, len(blockedUsers))
|
||||
blockedUserInfo := blockedUsers[0]
|
||||
assert.Equal(t, "u1", blockedUserInfo.ID)
|
||||
assert.True(t, blockedUserInfo.Until.After(time.Now().Add(time.Minute*59)), "blocked duration is more than 59m away")
|
||||
assert.True(t, blockedUserInfo.Until.Before(time.Now().Add(time.Minute*61)), "blocked duration is less than 61m away")
|
||||
assert.Equal(t, []interface{}{"u1"}, flags)
|
||||
}
|
||||
|
||||
func TestRPC_userDetailHndl(t *testing.T) {
|
||||
@@ -338,6 +301,6 @@ func TestRPC_closeHndl(t *testing.T) {
|
||||
api := fmt.Sprintf("http://localhost:%d/test", port)
|
||||
|
||||
re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
|
||||
assert.NoError(t, re.Close())
|
||||
assert.NoError(t, re.Close(), "second call should not result in panic or errors")
|
||||
err := re.Close()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
@@ -28,15 +28,6 @@ func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Resp
|
||||
return jrpc.EncodeResponse(id, nil, err)
|
||||
}
|
||||
|
||||
func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var fileID string
|
||||
if err := json.Unmarshal(params, &fileID); err != nil {
|
||||
return jrpc.Response{Error: err.Error()}
|
||||
}
|
||||
err := s.img.ResetCleanupTimer(fileID)
|
||||
return jrpc.EncodeResponse(id, nil, err)
|
||||
}
|
||||
|
||||
func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var fileID string
|
||||
if err := json.Unmarshal(params, &fileID); err != nil {
|
||||
@@ -46,16 +37,6 @@ func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response)
|
||||
return jrpc.EncodeResponse(id, value, err)
|
||||
}
|
||||
|
||||
func (s *RPC) imgDeleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var fileID string
|
||||
if err := json.Unmarshal(params, &fileID); err != nil {
|
||||
return jrpc.Response{Error: err.Error()}
|
||||
}
|
||||
err := s.img.Delete(fileID)
|
||||
return jrpc.EncodeResponse(id, nil, err)
|
||||
|
||||
}
|
||||
|
||||
func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
|
||||
var fileID string
|
||||
if err := json.Unmarshal(params, &fileID); err != nil {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
"github.com/go-pkgz/jrpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/image"
|
||||
"github.com/umputun/remark/backend/app/store/image"
|
||||
)
|
||||
|
||||
// gopher png for test, from https://golang.org/src/image/png/example_test.go
|
||||
@@ -45,7 +46,7 @@ const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwU
|
||||
|
||||
func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
|
||||
func gopherPNGBytes() []byte {
|
||||
img, _ := io.ReadAll(gopherPNG())
|
||||
img, _ := ioutil.ReadAll(gopherPNG())
|
||||
return img
|
||||
}
|
||||
|
||||
@@ -115,24 +116,7 @@ func TestRPC_imgCleanupHndl(t *testing.T) {
|
||||
assert.Equal(t, 1462, len(img))
|
||||
assert.Equal(t, gopherPNGBytes(), img)
|
||||
|
||||
// age the image past the ttl used below, so the reset that follows is what keeps it on
|
||||
// staging rather than the image simply being young
|
||||
const stagingTTL = 500 * time.Millisecond
|
||||
time.Sleep(stagingTTL + 100*time.Millisecond)
|
||||
|
||||
// reset the time to cleanup, which leaves a full ttl before it could be collected again
|
||||
err = ri.ResetCleanupTimer(id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// cleanup, should not affect the new image
|
||||
err = ri.Cleanup(context.TODO(), stagingTTL)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// load after cleanup should succeed
|
||||
_, err = ri.Load(id)
|
||||
assert.NoError(t, err, "image is still on staging because it's cleanup timer was reset")
|
||||
|
||||
// cleanup with short TTL, should remove the image from staging
|
||||
// cleanup
|
||||
err = ri.Cleanup(context.TODO(), time.Nanosecond)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -161,9 +145,4 @@ func TestRPC_imgInfoHndl(t *testing.T) {
|
||||
info, err = ri.Info()
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, info.FirstStagingImageTS.IsZero())
|
||||
|
||||
err = ri.Delete("test_img")
|
||||
assert.NoError(t, err)
|
||||
_, err = ri.Load("test_img")
|
||||
assert.EqualError(t, err, "image test_img not found")
|
||||
}
|
||||
|
||||
@@ -9,9 +9,9 @@ package server
|
||||
import (
|
||||
"github.com/go-pkgz/jrpc"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/image"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store/image"
|
||||
)
|
||||
|
||||
// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
|
||||
@@ -57,12 +57,10 @@ func (s *RPC) addHandlers() {
|
||||
|
||||
// image store handlers
|
||||
s.Group("image", jrpc.HandlersGroup{
|
||||
"save_with_id": s.imgSaveWithIDHndl,
|
||||
"reset_cleanup_timer": s.imgResetClnTimerHndl,
|
||||
"load": s.imgLoadHndl,
|
||||
"delete": s.imgDeleteHndl,
|
||||
"commit": s.imgCommitHndl,
|
||||
"cleanup": s.imgCleanupHndl,
|
||||
"info": s.imgInfoHndl,
|
||||
"save_with_id": s.imgSaveWithIDHndl,
|
||||
"load": s.imgLoadHndl,
|
||||
"commit": s.imgCommitHndl,
|
||||
"cleanup": s.imgCleanupHndl,
|
||||
"info": s.imgInfoHndl,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
@@ -16,41 +17,37 @@ import (
|
||||
"github.com/go-pkgz/jrpc"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/memory_store/accessor"
|
||||
"github.com/umputun/remark/memory_store/accessor"
|
||||
)
|
||||
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for i := 0; i < 10; i++ {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
// waitForHTTPServerStart blocks until the server on port answers, failing the test naming the
|
||||
// port if it never does
|
||||
func waitForHTTPServerStart(t *testing.T, port int) {
|
||||
t.Helper()
|
||||
func waitForHTTPServerStart(port int) {
|
||||
// wait for up to 3 seconds for server to start before returning it
|
||||
client := http.Client{Timeout: time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
for i := 0; i < 300; i++ {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}, 30*time.Second, 10*time.Millisecond, "http server on port %d didn't start", port)
|
||||
}
|
||||
}
|
||||
|
||||
func prepTestStore(t *testing.T) (port int, teardown func()) {
|
||||
mg := accessor.NewMemData()
|
||||
adm := accessor.NewMemAdminStore("secret")
|
||||
img := accessor.NewMemImageStore()
|
||||
s := NewRPC(mg, adm, img, jrpc.NewServer("/test"))
|
||||
s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
|
||||
|
||||
admRec := accessor.AdminRec{
|
||||
SiteID: "test-site",
|
||||
@@ -64,17 +61,14 @@ func prepTestStore(t *testing.T) (port int, teardown func()) {
|
||||
admRecDisabled.Enabled = false
|
||||
adm.Set("test-site-disabled", admRecDisabled)
|
||||
|
||||
port = chooseUnusedPort(t)
|
||||
port = chooseRandomUnusedPort()
|
||||
go func() {
|
||||
_ = s.Run(port)
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(t, port)
|
||||
waitForHTTPServerStart(port)
|
||||
|
||||
return port, func() {
|
||||
// every test client here uses http.DefaultTransport, so their keep-alive connections
|
||||
// sit in one shared pool; Shutdown waits on them and hits its own 5s deadline otherwise
|
||||
http.DefaultTransport.(*http.Transport).CloseIdleConnections()
|
||||
require.NoError(t, s.Shutdown())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/go-pkgz/auth/v2/avatar"
|
||||
"github.com/go-pkgz/auth/avatar"
|
||||
)
|
||||
|
||||
// AvatarCommand set of flags and command for avatar migration
|
||||
@@ -39,12 +39,12 @@ func (ac *AvatarCommand) Execute(_ []string) error {
|
||||
|
||||
src, err := ac.makeAvatarStore(ac.AvatarSrc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't make avatart store for %s: %w", ac.AvatarSrc.Type, err)
|
||||
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
|
||||
}
|
||||
|
||||
dst, err := ac.makeAvatarStore(ac.AvatarDst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't make avatart store for %s: %w", ac.AvatarDst.Type, err)
|
||||
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
|
||||
}
|
||||
|
||||
if ac.migrator == nil {
|
||||
@@ -72,14 +72,14 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
|
||||
switch gr.Type {
|
||||
case "fs":
|
||||
if err := makeDirs(gr.FS.Path); err != nil {
|
||||
return nil, fmt.Errorf("failed to create avatar store: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return avatar.NewLocalFS(gr.FS.Path), nil
|
||||
case "bolt":
|
||||
if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
|
||||
return nil, fmt.Errorf("failed to create avatar store: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported avatar store type %s", gr.Type)
|
||||
return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
|
||||
}
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/go-pkgz/auth/v2/avatar"
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/go-pkgz/auth/avatar"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/umputun/go-flags"
|
||||
)
|
||||
|
||||
func TestAvatar_Execute(t *testing.T) {
|
||||
|
||||
defer os.RemoveAll("/tmp/ava-test")
|
||||
|
||||
// from fs to bolt
|
||||
@@ -21,18 +22,16 @@ func TestAvatar_Execute(t *testing.T) {
|
||||
_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
|
||||
"--dst.bolt.file=/tmp/ava-test.db"})
|
||||
require.NoError(t, err)
|
||||
defer os.Remove("/tmp/ava-test.db")
|
||||
err = cmd.Execute(nil)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// failed
|
||||
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: fmt.Errorf("failed blah")}}
|
||||
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
|
||||
p = flags.NewParser(&cmd, flags.Default)
|
||||
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
|
||||
"--dst.bolt.file=/tmp/ava-test2.db"})
|
||||
require.NoError(t, err)
|
||||
defer os.Remove("/tmp/ava-test2.db")
|
||||
err = cmd.Execute(nil)
|
||||
assert.Error(t, err, "failed blah")
|
||||
}
|
||||
|
||||
+14
-13
@@ -9,15 +9,17 @@ import (
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// BackupCommand set of flags and command for export
|
||||
// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
|
||||
type BackupCommand struct {
|
||||
ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
|
||||
ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
|
||||
|
||||
SupportCmdOpts
|
||||
ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
|
||||
ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
Timeout time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
@@ -36,20 +38,19 @@ func (ec *BackupCommand) Execute(_ []string) error {
|
||||
|
||||
// prepare http client and request
|
||||
client := http.Client{}
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
|
||||
defer cancel()
|
||||
exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s", ec.RemarkURL, ec.Site)
|
||||
req, err := http.NewRequest(http.MethodGet, exportURL, http.NoBody)
|
||||
req, err := http.NewRequest(http.MethodGet, exportURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't make export request for %s: %w", exportURL, err)
|
||||
return errors.Wrapf(err, "can't make export request for %s", exportURL)
|
||||
}
|
||||
req.SetBasicAuth("admin", ec.AdminPasswd)
|
||||
|
||||
// get with timeout
|
||||
resp, err := client.Do(req.WithContext(ctx)) //nolint:gosec // exportURL is built from operator-supplied CLI flags, not user input
|
||||
resp, err := client.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed for %s: %w", exportURL, err)
|
||||
return errors.Wrapf(err, "request failed for %s", exportURL)
|
||||
}
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
@@ -61,18 +62,18 @@ func (ec *BackupCommand) Execute(_ []string) error {
|
||||
return responseError(resp)
|
||||
}
|
||||
|
||||
fh, err := os.Create(fname) //nolint:gosec // harmless
|
||||
fh, err := os.Create(fname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't create backup file %s: %w", fname, err)
|
||||
return errors.Wrapf(err, "can't create backup file %s", fname)
|
||||
}
|
||||
defer func() { //nolint:gosec // false positive on defer without error check when it's checked here
|
||||
defer func() {
|
||||
if err = fh.Close(); err != nil {
|
||||
log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err = io.Copy(fh, resp.Body); err != nil {
|
||||
return fmt.Errorf("failed to write backup file %s: %w", fname, err)
|
||||
return errors.Wrapf(err, "failed to write backup file %s", fname)
|
||||
}
|
||||
|
||||
log.Printf("[INFO] export completed, file %s", fname)
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -18,10 +18,6 @@ func TestBackup_Execute(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
|
||||
assert.Equal(t, "GET", r.Method)
|
||||
t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "admin:secret", string(auth))
|
||||
fmt.Fprint(w, "blah\nblah2\n12345678\n")
|
||||
}))
|
||||
defer ts.Close()
|
||||
@@ -35,33 +31,11 @@ func TestBackup_Execute(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
defer os.Remove("/tmp/remark-test.export")
|
||||
|
||||
data, err := os.ReadFile("/tmp/remark-test.export")
|
||||
data, err := ioutil.ReadFile("/tmp/remark-test.export")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
|
||||
}
|
||||
|
||||
func TestBackup_ExecuteNoPassword(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
|
||||
assert.Equal(t, "GET", r.Method)
|
||||
t.Logf("Authorization: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "admin:", string(auth))
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
fmt.Fprint(w, "Unauthorized")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cmd := BackupCommand{}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
|
||||
p := flags.NewParser(&cmd, flags.Default)
|
||||
_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
|
||||
require.NoError(t, err)
|
||||
err = cmd.Execute(nil)
|
||||
assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
|
||||
}
|
||||
|
||||
func TestBackup_ExecuteFailedStatus(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
|
||||
|
||||
+41
-39
@@ -9,20 +9,21 @@ import (
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
// CleanupCommand set of flags and command for cleanup
|
||||
type CleanupCommand struct {
|
||||
Dry bool `long:"dry" description:"dry mode, will not remove comments"`
|
||||
From string `long:"from" description:"from yyyymmdd"`
|
||||
To string `long:"to" description:"from yyyymmdd"`
|
||||
BadWords []string `short:"w" long:"bword" description:"bad word(s)"`
|
||||
BadUsers []string `short:"u" long:"buser" description:"bad user(s)"`
|
||||
SetTitle bool `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
|
||||
|
||||
SupportCmdOpts
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
Dry bool `long:"dry" description:"dry mode, will not remove comments"`
|
||||
From string `long:"from" description:"from yyyymmdd"`
|
||||
To string `long:"to" description:"from yyyymmdd"`
|
||||
BadWords []string `short:"w" long:"bword" description:"bad word(s)"`
|
||||
BadUsers []string `short:"u" long:"buser" description:"bad user(s)"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
SetTitle bool `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
@@ -38,7 +39,7 @@ func (cc *CleanupCommand) Execute(_ []string) error {
|
||||
|
||||
posts, err := cc.postsInRange(cc.From, cc.To)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get posts: %w", err)
|
||||
return errors.Wrap(err, "can't get posts")
|
||||
}
|
||||
log.Printf("[DEBUG] got %d posts", len(posts))
|
||||
|
||||
@@ -54,6 +55,7 @@ func (cc *CleanupCommand) Execute(_ []string) error {
|
||||
cc.procTitles(comments)
|
||||
} else {
|
||||
spamComments += cc.procSpam(comments)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,7 +79,7 @@ func (cc *CleanupCommand) procSpam(comments []store.Comment) int {
|
||||
log.Printf("[WARN] can't remove comment, %v", err)
|
||||
}
|
||||
}
|
||||
comment.Text = strings.ReplaceAll(comment.Text, "\n", " ")
|
||||
comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
|
||||
log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
|
||||
}
|
||||
}
|
||||
@@ -98,7 +100,7 @@ func (cc *CleanupCommand) procTitles(comments []store.Comment) {
|
||||
func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, error) {
|
||||
posts, err := cc.listPosts()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't list posts for %s: %w", cc.Site, err)
|
||||
return nil, errors.Wrapf(err, "can't list posts for %s", cc.Site)
|
||||
}
|
||||
|
||||
from, to := defaultFrom, defaultTo
|
||||
@@ -106,14 +108,14 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
|
||||
if fromS != "" {
|
||||
from, err = time.ParseInLocation("20060102", fromS, time.Local)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't parse --from: %w", err)
|
||||
return nil, errors.Wrap(err, "can't parse --from")
|
||||
}
|
||||
}
|
||||
|
||||
if toS != "" {
|
||||
to, err = time.ParseInLocation("20060102", toS, time.Local)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't parse --to: %w", err)
|
||||
return nil, errors.Wrap(err, "can't parse --to")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,38 +132,37 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
|
||||
func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
|
||||
listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
|
||||
client := http.Client{Timeout: 30 * time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
r, err := client.Get(listURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get request failed for list of posts, site %s: %w", cc.Site, err)
|
||||
return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
|
||||
}
|
||||
defer func() { _ = r.Body.Close() }()
|
||||
|
||||
if r.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("request %s failed with status %d", listURL, r.StatusCode)
|
||||
return nil, errors.Errorf("request %s failed with status %d", listURL, r.StatusCode)
|
||||
}
|
||||
|
||||
list := []store.PostInfo{}
|
||||
if err = json.NewDecoder(r.Body).Decode(&list); err != nil {
|
||||
return nil, fmt.Errorf("can't decode list of posts for site %s: %w", cc.Site, err)
|
||||
return nil, errors.Wrapf(err, "can't decode list of posts for site %s", cc.Site)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// get all comments for post url via /find?site=siteID&url=post-url&format=[tree|plain]
|
||||
func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error) {
|
||||
|
||||
commentsURL := fmt.Sprintf("%s/api/v1/find?site=%s&url=%s&format=plain", cc.RemarkURL, cc.Site, postURL)
|
||||
|
||||
var r *http.Response
|
||||
var err error
|
||||
|
||||
// handle 429 error from limiter
|
||||
client := http.Client{Timeout: 30 * time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
for {
|
||||
client := http.Client{Timeout: 30 * time.Second}
|
||||
r, err = client.Get(commentsURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get request failed for comments, %s: %w", postURL, err)
|
||||
return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
|
||||
}
|
||||
if r.StatusCode == http.StatusTooManyRequests {
|
||||
_ = r.Body.Close()
|
||||
@@ -174,66 +175,67 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
|
||||
defer func() { _ = r.Body.Close() }()
|
||||
|
||||
if r.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
|
||||
return nil, errors.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
|
||||
}
|
||||
|
||||
commentsWithInfo := struct {
|
||||
Comments []store.Comment `json:"comments"`
|
||||
Info store.PostInfo `json:"info"`
|
||||
Info store.PostInfo `json:"info,omitempty"`
|
||||
}{}
|
||||
|
||||
if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
|
||||
return nil, fmt.Errorf("can't decode list of comments for %s: %w", postURL, err)
|
||||
return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
|
||||
}
|
||||
return commentsWithInfo.Comments, nil
|
||||
}
|
||||
|
||||
// deleteComment with DELETE /admin/comment/{id}?site=siteID&url=post-url
|
||||
func (cc *CleanupCommand) deleteComment(c store.Comment) error { //nolint:dupl // not worth combining
|
||||
func (cc *CleanupCommand) deleteComment(c store.Comment) error {
|
||||
|
||||
deleteURL := fmt.Sprintf("%s/api/v1/admin/comment/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
|
||||
req, err := http.NewRequest("DELETE", deleteURL, http.NoBody)
|
||||
req, err := http.NewRequest("DELETE", deleteURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to make delete request for comment %s, %s: %w", c.ID, c.Locator.URL, err)
|
||||
return errors.Wrapf(err, "failed to make delete request for comment %s, %s", c.ID, c.Locator.URL)
|
||||
}
|
||||
req.SetBasicAuth("admin", cc.AdminPasswd)
|
||||
|
||||
client := http.Client{}
|
||||
defer client.CloseIdleConnections()
|
||||
r, err := client.Do(req) //nolint:gosec // RemarkURL comes from operator CLI flag, not user input
|
||||
r, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete request failed for comment %s, %s: %w", c.ID, c.Locator.URL, err)
|
||||
return errors.Wrapf(err, "delete request failed for comment %s, %s", c.ID, c.Locator.URL)
|
||||
}
|
||||
defer func() { _ = r.Body.Close() }()
|
||||
if r.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("delete request failed with status %s", r.Status)
|
||||
return errors.Errorf("delete request failed with status %s", r.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
|
||||
func (cc *CleanupCommand) setTitle(c store.Comment) error { //nolint:dupl // not worth combining
|
||||
func (cc *CleanupCommand) setTitle(c store.Comment) error {
|
||||
|
||||
titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
|
||||
req, err := http.NewRequest("PUT", titleURL, http.NoBody)
|
||||
req, err := http.NewRequest("PUT", titleURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to make title request for comment %s, %s: %w", c.ID, c.Locator.URL, err)
|
||||
return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
|
||||
}
|
||||
req.SetBasicAuth("admin", cc.AdminPasswd)
|
||||
|
||||
client := http.Client{}
|
||||
defer client.CloseIdleConnections()
|
||||
r, err := client.Do(req) //nolint:gosec // RemarkURL comes from operator CLI flag, not user input
|
||||
r, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("title request failed for comment %s, %s: %w", c.ID, c.Locator.URL, err)
|
||||
return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
|
||||
}
|
||||
defer func() { _ = r.Body.Close() }()
|
||||
if r.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("title request failed with status %s", r.Status)
|
||||
return errors.Errorf("title request failed with status %s", r.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isSpam calculates spam's probability as a score
|
||||
func (cc *CleanupCommand) isSpam(comment store.Comment) (isSpam bool, spamScore float64) {
|
||||
func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
|
||||
|
||||
badWord := func(txt string) float64 {
|
||||
res := 0.0
|
||||
for _, w := range cc.BadWords {
|
||||
|
||||
@@ -9,11 +9,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
type cleanedComments struct {
|
||||
@@ -45,6 +46,7 @@ func TestCleanup_IsSpam(t *testing.T) {
|
||||
}
|
||||
|
||||
for n, tt := range tbl {
|
||||
tt := tt
|
||||
checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
|
||||
t.Run(checkName, func(t *testing.T) {
|
||||
c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
|
||||
@@ -57,7 +59,8 @@ func TestCleanup_IsSpam(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCleanup_postsInRange(t *testing.T) {
|
||||
r := http.NewServeMux()
|
||||
|
||||
r := chi.NewRouter()
|
||||
cleanupRoutes(t, r, nil)
|
||||
ts := httptest.NewServer(r)
|
||||
defer ts.Close()
|
||||
@@ -80,7 +83,7 @@ func TestCleanup_postsInRange(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCleanup_listComments(t *testing.T) {
|
||||
r := http.NewServeMux()
|
||||
r := chi.NewRouter()
|
||||
cleanupRoutes(t, r, nil)
|
||||
ts := httptest.NewServer(r)
|
||||
defer ts.Close()
|
||||
@@ -106,7 +109,7 @@ func TestCleanup_listComments(t *testing.T) {
|
||||
|
||||
func TestCleanup_ExecuteSpam(t *testing.T) {
|
||||
cleaned := cleanedComments{}
|
||||
r := http.NewServeMux()
|
||||
r := chi.NewRouter()
|
||||
cleanupRoutes(t, r, &cleaned)
|
||||
ts := httptest.NewServer(r)
|
||||
defer ts.Close()
|
||||
@@ -125,7 +128,7 @@ func TestCleanup_ExecuteSpam(t *testing.T) {
|
||||
|
||||
func TestCleanup_ExecuteTitle(t *testing.T) {
|
||||
titledComments := cleanedComments{}
|
||||
r := http.NewServeMux()
|
||||
r := chi.NewRouter()
|
||||
cleanupRoutes(t, r, &titledComments)
|
||||
ts := httptest.NewServer(r)
|
||||
defer ts.Close()
|
||||
@@ -141,7 +144,7 @@ func TestCleanup_ExecuteTitle(t *testing.T) {
|
||||
assert.Equal(t, []string{"/api/v1/admin/title/1", "/api/v1/admin/title/2", "/api/v1/admin/title/3", "/api/v1/admin/title/11"}, titledComments.ids)
|
||||
}
|
||||
|
||||
func cleanupRoutes(t *testing.T, r *http.ServeMux, c *cleanedComments) {
|
||||
func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
|
||||
r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, "GET", r.Method)
|
||||
require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
|
||||
@@ -172,7 +175,7 @@ func cleanupRoutes(t *testing.T, r *http.ServeMux, c *cleanedComments) {
|
||||
|
||||
commentsWithInfo := struct {
|
||||
Comments []store.Comment `json:"comments"`
|
||||
Info store.PostInfo `json:"info"`
|
||||
Info store.PostInfo `json:"info,omitempty"`
|
||||
}{}
|
||||
|
||||
switch r.URL.Query().Get("url") {
|
||||
@@ -193,7 +196,7 @@ func cleanupRoutes(t *testing.T, r *http.ServeMux, c *cleanedComments) {
|
||||
require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
|
||||
})
|
||||
|
||||
r.HandleFunc("/api/v1/admin/comment/{id}", func(_ http.ResponseWriter, r *http.Request) {
|
||||
r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
t.Log("delete ", r.URL.Path)
|
||||
c.lock.Lock()
|
||||
@@ -201,11 +204,12 @@ func cleanupRoutes(t *testing.T, r *http.ServeMux, c *cleanedComments) {
|
||||
c.lock.Unlock()
|
||||
})
|
||||
|
||||
r.HandleFunc("/api/v1/admin/title/{id}", func(_ http.ResponseWriter, r *http.Request) {
|
||||
r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, "PUT", r.Method)
|
||||
t.Log("title for ", r.URL.Path)
|
||||
c.lock.Lock()
|
||||
c.ids = append(c.ids, r.URL.Path)
|
||||
c.lock.Unlock()
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
+11
-22
@@ -4,8 +4,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -14,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// CommonOptionsCommander extends flags.Commander with SetCommon
|
||||
@@ -31,20 +31,11 @@ type CommonOpts struct {
|
||||
Revision string
|
||||
}
|
||||
|
||||
// SupportCmdOpts is set of commands shared among similar commands like backup/restore and such.
|
||||
// Order of fields defines the help command output order.
|
||||
type SupportCmdOpts struct {
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
|
||||
Timeout time.Duration `long:"timeout" default:"60m" description:"timeout for the command run"`
|
||||
}
|
||||
|
||||
// DeprecatedFlag contains information about deprecated option
|
||||
type DeprecatedFlag struct {
|
||||
Old string
|
||||
New string
|
||||
Version string
|
||||
Collision bool
|
||||
Old string
|
||||
New string
|
||||
RemoveVersion string
|
||||
}
|
||||
|
||||
// SetCommon satisfies CommonOptionsCommander interface and sets common option fields
|
||||
@@ -67,6 +58,7 @@ type fileParser struct {
|
||||
|
||||
// parse apply template and also concat path and file. In case if file contains path separator path will be ignored
|
||||
func (p *fileParser) parse(now time.Time) (string, error) {
|
||||
|
||||
// file/location parameters my have template masks
|
||||
fileTemplate := struct {
|
||||
YYYYMMDD string
|
||||
@@ -95,7 +87,7 @@ func (p *fileParser) parse(now time.Time) (string, error) {
|
||||
}
|
||||
|
||||
if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
|
||||
return "", fmt.Errorf("failed to parse %q: %w", fname, err)
|
||||
return "", errors.Wrapf(err, "failed to parse %q", fname)
|
||||
}
|
||||
return bb.String(), nil
|
||||
}
|
||||
@@ -111,21 +103,18 @@ func resetEnv(envs ...string) {
|
||||
|
||||
// responseError returns error with status and response body
|
||||
func responseError(resp *http.Response) error {
|
||||
body, e := io.ReadAll(resp.Body)
|
||||
body, e := ioutil.ReadAll(resp.Body)
|
||||
if e != nil {
|
||||
body = []byte("")
|
||||
}
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
return fmt.Errorf("error response %q, ensure you have set ADMIN_PASSWD and provided it to the command you're running: %s", resp.Status, body)
|
||||
}
|
||||
return fmt.Errorf("error response %q, %s", resp.Status, body)
|
||||
return errors.Errorf("error response %q, %s", resp.Status, body)
|
||||
}
|
||||
|
||||
// mkdir -p for all dirs
|
||||
func makeDirs(dirs ...string) error {
|
||||
for _, dir := range dirs {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil { // if path is already a directory, MkdirAll does nothing
|
||||
return fmt.Errorf("can't make directory %s: %w", dir, err)
|
||||
if err := os.MkdirAll(dir, 0700); err != nil { // If path is already a directory, MkdirAll does nothing
|
||||
return errors.Wrapf(err, "can't make directory %s", dir)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
+16
-13
@@ -5,19 +5,23 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ImportCommand set of flags and command for import
|
||||
type ImportCommand struct {
|
||||
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
|
||||
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" choice:"commento" description:"import format"` //nolint
|
||||
|
||||
SupportCmdOpts
|
||||
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
|
||||
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"` //nolint
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
@@ -28,23 +32,22 @@ func (ic *ImportCommand) Execute(_ []string) error {
|
||||
|
||||
reader, err := ic.reader(ic.InputFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't open import file %s: %w", ic.InputFile, err)
|
||||
return errors.Wrapf(err, "can't open import file %s", ic.InputFile)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
|
||||
defer cancel()
|
||||
importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s", ic.RemarkURL, ic.Site, ic.Provider)
|
||||
req, err := http.NewRequest(http.MethodPost, importURL, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't make import request for %s: %w", importURL, err)
|
||||
return errors.Wrapf(err, "can't make import request for %s", importURL)
|
||||
}
|
||||
req.SetBasicAuth("admin", ic.AdminPasswd)
|
||||
|
||||
resp, err := client.Do(req.WithContext(ctx)) //nolint:gosec // importURL built from operator CLI flags, not user input; closes request's reader
|
||||
resp, err := client.Do(req.WithContext(ctx)) // closes request's reader
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed for %s: %w", importURL, err)
|
||||
return errors.Wrapf(err, "request failed for %s", importURL)
|
||||
}
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
@@ -55,9 +58,9 @@ func (ic *ImportCommand) Execute(_ []string) error {
|
||||
return responseError(resp)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get response from importer: %w", err)
|
||||
return errors.Wrap(err, "can't get response from importer")
|
||||
}
|
||||
|
||||
log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
|
||||
@@ -68,13 +71,13 @@ func (ic *ImportCommand) Execute(_ []string) error {
|
||||
func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
|
||||
inpFile, err := os.Open(inp) // nolint
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("import failed, can't open %s: %w", inp, err)
|
||||
return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
|
||||
}
|
||||
|
||||
reader = inpFile
|
||||
if strings.HasSuffix(ic.InputFile, ".gz") {
|
||||
if reader, err = gzip.NewReader(inpFile); err != nil {
|
||||
return nil, fmt.Errorf("can't make gz reader: %w", err)
|
||||
return nil, errors.Wrap(err, "can't make gz reader")
|
||||
}
|
||||
}
|
||||
return reader, nil
|
||||
|
||||
@@ -1,29 +1,26 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestImport_Execute(t *testing.T) {
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "admin:secret", string(auth))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
|
||||
|
||||
@@ -51,43 +48,8 @@ func TestImport_Execute(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestImport_ExecuteNoPassword(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "admin:", string(auth))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
|
||||
|
||||
w.WriteHeader(401)
|
||||
fmt.Fprint(w, "Unauthorized")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cmd := ImportCommand{}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
|
||||
|
||||
p := flags.NewParser(&cmd, flags.Default)
|
||||
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
|
||||
require.NoError(t, err)
|
||||
err = cmd.Execute(nil)
|
||||
assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
|
||||
|
||||
cmd = ImportCommand{}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
|
||||
|
||||
p = flags.NewParser(&cmd, flags.Default)
|
||||
_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"})
|
||||
require.NoError(t, err)
|
||||
err = cmd.Execute(nil)
|
||||
assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
|
||||
}
|
||||
|
||||
func TestImport_ExecuteFailed(t *testing.T) {
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
@@ -132,14 +94,16 @@ func TestImport_ExecuteFailed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestImport_ExecuteTimeout(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
body, err := io.ReadAll(r.Body)
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
|
||||
// hold the response until the client gives up on its own timeout
|
||||
<-r.Context().Done()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
fmt.Fprintln(w, "some response")
|
||||
fmt.Fprintln(w, string(body))
|
||||
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
|
||||
+14
-12
@@ -3,19 +3,22 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// RemapCommand set of flags and command for change linkage between comments to
|
||||
// different urls based on given rules (input file)
|
||||
type RemapCommand struct {
|
||||
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
|
||||
|
||||
SupportCmdOpts
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
Timeout time.Duration `long:"timeout" default:"15m" description:"remap timeout"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
@@ -26,23 +29,22 @@ func (rc *RemapCommand) Execute(_ []string) error {
|
||||
|
||||
rulesReader, err := os.Open(rc.InputFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant open file %s: %w", rc.InputFile, err)
|
||||
return errors.Wrapf(err, "cant open file %s", rc.InputFile)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rc.Timeout)
|
||||
defer cancel()
|
||||
remapURL := fmt.Sprintf("%s/api/v1/admin/remap?site=%s", rc.RemarkURL, rc.Site)
|
||||
req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader) //nolint:gosec // RemarkURL is operator CLI flag, not user input
|
||||
req, err := http.NewRequest(http.MethodPost, remapURL, rulesReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't make remap request for %s: %w", remapURL, err)
|
||||
return errors.Wrapf(err, "can't make remap request for %s", remapURL)
|
||||
}
|
||||
req.SetBasicAuth("admin", rc.AdminPasswd)
|
||||
|
||||
resp, err := client.Do(req.WithContext(ctx)) //nolint:gosec // see above
|
||||
resp, err := client.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed for %s: %w", remapURL, err)
|
||||
return errors.Wrapf(err, "request failed for %s", remapURL)
|
||||
}
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
@@ -53,9 +55,9 @@ func (rc *RemapCommand) Execute(_ []string) error {
|
||||
return responseError(resp)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get response: %w", err)
|
||||
return errors.Wrap(err, "can't get response")
|
||||
}
|
||||
|
||||
log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
|
||||
|
||||
@@ -1,29 +1,24 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRemap_Execute(t *testing.T) {
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
assert.Equal(t, "remark", r.URL.Query().Get("site"))
|
||||
t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "admin:secret", string(auth))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
|
||||
|
||||
@@ -40,31 +35,3 @@ func TestRemap_Execute(t *testing.T) {
|
||||
err = cmd.Execute(nil)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestRemap_ExecuteNoPassword(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/remap")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
assert.Equal(t, "remark", r.URL.Query().Get("site"))
|
||||
t.Logf("Authorization header: %+v", r.Header.Get("Authorization"))
|
||||
auth, err := base64.StdEncoding.DecodeString(strings.Split(r.Header.Get("Authorization"), " ")[1])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "admin:", string(auth))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "http://oldsite.com* https://newsite.com*\nhttp://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1", string(body))
|
||||
|
||||
w.WriteHeader(401)
|
||||
fmt.Fprint(w, "Unauthorized")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cmd := RemapCommand{}
|
||||
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL})
|
||||
|
||||
p := flags.NewParser(&cmd, flags.Default)
|
||||
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/remap_urls.txt"})
|
||||
require.NoError(t, err)
|
||||
err = cmd.Execute(nil)
|
||||
assert.EqualError(t, err, "error response \"401 Unauthorized\", ensure you have set ADMIN_PASSWD and provided it to the command you're running: Unauthorized")
|
||||
}
|
||||
|
||||
@@ -11,7 +11,9 @@ type RestoreCommand struct {
|
||||
ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
|
||||
ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
|
||||
|
||||
SupportCmdOpts
|
||||
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
|
||||
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
|
||||
AdminPasswd string `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
|
||||
CommonOpts
|
||||
}
|
||||
|
||||
@@ -27,10 +29,12 @@ func (rc *RestoreCommand) Execute(args []string) error {
|
||||
return err
|
||||
}
|
||||
importer := ImportCommand{
|
||||
InputFile: fname,
|
||||
Provider: "native",
|
||||
SupportCmdOpts: rc.SupportCmdOpts,
|
||||
CommonOpts: rc.CommonOpts,
|
||||
InputFile: fname,
|
||||
Site: rc.Site,
|
||||
Provider: "native",
|
||||
Timeout: rc.Timeout,
|
||||
AdminPasswd: rc.AdminPasswd,
|
||||
CommonOpts: rc.CommonOpts,
|
||||
}
|
||||
return importer.Execute(args)
|
||||
}
|
||||
|
||||
@@ -2,22 +2,24 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRestore_Execute(t *testing.T) {
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
assert.Equal(t, "native", r.URL.Query().Get("provider"))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
|
||||
|
||||
|
||||
+330
-958
File diff suppressed because it is too large
Load Diff
+161
-765
File diff suppressed because it is too large
Load Diff
-16
@@ -1,16 +0,0 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKNwapOQ6rQJHetP
|
||||
HRlJBIh1OsOsUBiXb3rXXE3xpWAxAha0MH+UPRblOko+5T2JqIb+xKf9Vi3oTM3t
|
||||
KvffaOPtzKXZauscjq6NGzA3LgeiMy6q19pvkUUOlGYK6+Xfl+B7Xw6+hBMkQuGE
|
||||
nUS8nkpR5mK4ne7djIyfHFfMu4ptAgMBAAECgYA+s0PPtMq1osG9oi4xoxeAGikf
|
||||
JB3eMUptP+2DYW7mRibc+ueYKhB9lhcUoKhlQUhL8bUUFVZYakP8xD21thmQqnC4
|
||||
f63asad0ycteJMLb3r+z26LHuCyOdPg1pyLk3oQ32lVQHBCYathRMcVznxOG16VK
|
||||
I8BFfstJTaJu0lK/wQJBANYFGusBiZsJQ3utrQMVPpKmloO2++4q1v6ZR4puDQHx
|
||||
TjLjAIgrkYfwTJBLBRZxec0E7TmuVQ9uJ+wMu/+7zaUCQQDDf2xMnQqYknJoKGq+
|
||||
oAnyC66UqWC5xAnQS32mlnJ632JXA0pf9pb1SXAYExB1p9Dfqd3VAwQDwBsDDgP6
|
||||
HD8pAkEA0lscNQZC2TaGtKZk2hXkdcH1SKru/g3vWTkRHxfCAznJUaza1fx0wzdG
|
||||
GcES1Bdez0tbW4llI5By/skZc2eE3QJAFl6fOskBbGHde3Oce0F+wdZ6XIJhEgCP
|
||||
iukIcKZoZQzoiMJUoVRrA5gqnmaYDI5uRRl/y57zt6YksR3KcLUIuQJAd242M/WF
|
||||
6YAZat3q/wEeETeQq1wrooew+8lHl05/Nt0cCpV48RGEhJ83pzBm3mnwHf8lTBJH
|
||||
x6XroMXsmbnsEw==
|
||||
-----END PRIVATE KEY-----
|
||||
Vendored
-6
@@ -1,6 +0,0 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgGH2MylyZjjRdauTk
|
||||
xxXW6p8VSHqIeVRRKSJPg1xn6+KgCgYIKoZIzj0DAQehRANCAAS/mNzQ7aBbIBr3
|
||||
DiHiJGIDEzi6+q3mmyhH6ZWQWFdFei2qgdyM1V6qtRPVq+yHBNSBebbR4noE/IYO
|
||||
hMdWYrKn
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -1 +0,0 @@
|
||||
This stub page would be replaced by the frontend statically built HTML during the Docker image build.
|
||||
+17
-29
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -9,9 +8,9 @@ import (
|
||||
"syscall"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/jessevdk/go-flags"
|
||||
"github.com/umputun/go-flags"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/cmd"
|
||||
"github.com/umputun/remark/backend/app/cmd"
|
||||
)
|
||||
|
||||
// Opts with all cli commands and flags
|
||||
@@ -24,9 +23,8 @@ type Opts struct {
|
||||
CleanupCmd cmd.CleanupCommand `command:"cleanup"`
|
||||
RemapCmd cmd.RemapCommand `command:"remap"`
|
||||
|
||||
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
|
||||
// SharedSecret is only used in server command, but defined for all commands for historical reasons
|
||||
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"the shared secret key used to sign JWT, should be a random, long, hard-to-guess string"`
|
||||
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
|
||||
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
|
||||
|
||||
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
|
||||
}
|
||||
@@ -47,7 +45,10 @@ func main() {
|
||||
SharedSecret: opts.SharedSecret,
|
||||
Revision: revision,
|
||||
})
|
||||
logDeprecatedParams(c.HandleDeprecatedFlags())
|
||||
for _, entry := range c.HandleDeprecatedFlags() {
|
||||
log.Printf("[WARN] --%s is deprecated and will be removed in v%s, please use --%s instead",
|
||||
entry.Old, entry.RemoveVersion, entry.New)
|
||||
}
|
||||
err := c.Execute(args)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] failed with %+v", err)
|
||||
@@ -56,11 +57,11 @@ func main() {
|
||||
}
|
||||
|
||||
if _, err := p.Parse(); err != nil {
|
||||
var flagsErr *flags.Error
|
||||
if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp {
|
||||
if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
|
||||
os.Exit(0)
|
||||
} else {
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,34 +73,21 @@ func setupLog(dbg bool) {
|
||||
log.Setup(log.Msec, log.LevelBraces)
|
||||
}
|
||||
|
||||
// logs usual and "collision" deprecated parameters
|
||||
func logDeprecatedParams(params []cmd.DeprecatedFlag) {
|
||||
for _, entry := range params {
|
||||
var deprecationNote string
|
||||
if entry.Collision {
|
||||
deprecationNote = fmt.Sprintf("[ERROR] deprecated --%s and new --%s options are set to different values, old one is ignored: please remove it", entry.Old, entry.New)
|
||||
} else {
|
||||
deprecationNote = fmt.Sprintf("[WARN] --%s is deprecated since v%s and will be removed in the future", entry.Old, entry.Version)
|
||||
if entry.New != "" {
|
||||
deprecationNote += fmt.Sprintf(", please use --%s instead", entry.New)
|
||||
}
|
||||
}
|
||||
log.Print(deprecationNote)
|
||||
}
|
||||
}
|
||||
|
||||
// getDump reads runtime stack and returns as a string
|
||||
func getDump() string {
|
||||
maxSize := 5 * 1024 * 1024
|
||||
stacktrace := make([]byte, maxSize)
|
||||
length := min(runtime.Stack(stacktrace, true), maxSize)
|
||||
length := runtime.Stack(stacktrace, true)
|
||||
if length > maxSize {
|
||||
length = maxSize
|
||||
}
|
||||
return string(stacktrace[:length])
|
||||
}
|
||||
|
||||
//nolint:gochecknoinits // can't avoid it in this place
|
||||
// nolint:gochecknoinits
|
||||
func init() {
|
||||
// catch SIGQUIT and print stack traces
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
sigChan := make(chan os.Signal)
|
||||
go func() {
|
||||
for range sigChan {
|
||||
log.Printf("[INFO] SIGQUIT detected, dump:\n%s", getDump())
|
||||
|
||||
+27
-112
@@ -2,29 +2,28 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func Test_Main(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.TempDir(), "remark42")
|
||||
|
||||
dir, err := ioutil.TempDir(os.TempDir(), "remark42")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
port := chooseUnusedPort(t)
|
||||
port := chooseRandomUnusedPort()
|
||||
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
|
||||
"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
|
||||
|
||||
@@ -47,127 +46,43 @@ func Test_Main(t *testing.T) {
|
||||
<-finished
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(t, port)
|
||||
waitForHTTPServerStart(port)
|
||||
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "pong", string(body))
|
||||
}
|
||||
|
||||
func TestMain_WithWebhook(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.TempDir(), "remark42")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
var webhookSent atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
webhookSent.Store(1)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
|
||||
b, e := io.ReadAll(r.Body)
|
||||
defer r.Body.Close()
|
||||
|
||||
assert.Nil(t, e)
|
||||
assert.Equal(t, "Comment: env test", string(b))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
port := chooseUnusedPort(t)
|
||||
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
|
||||
"--avatar.fs.path=" + dir, "--port=" + strconv.Itoa(port), "--url=https://demo.remark42.com", "--dbg",
|
||||
"--admin-passwd=password", "--site=remark", "--notify.admins=webhook"}
|
||||
|
||||
err = os.Setenv("NOTIFY_WEBHOOK_URL", ts.URL)
|
||||
assert.NoError(t, err)
|
||||
err = os.Setenv("NOTIFY_WEBHOOK_TEMPLATE", "Comment: {{.Orig}}")
|
||||
assert.NoError(t, err)
|
||||
err = os.Setenv("NOTIFY_WEBHOOK_HEADERS", "Content-Type:application/json")
|
||||
assert.NoError(t, err)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
<-done
|
||||
e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
|
||||
require.NoError(t, e)
|
||||
}()
|
||||
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
main()
|
||||
close(finished)
|
||||
}()
|
||||
|
||||
// defer cleanup because require check below can fail
|
||||
defer func() {
|
||||
close(done)
|
||||
<-finished
|
||||
}()
|
||||
|
||||
waitForHTTPServerStart(t, port)
|
||||
|
||||
resp, err := http.Post(fmt.Sprintf("http://admin:password@localhost:%d/api/v1/comment", port), "",
|
||||
strings.NewReader(`{"text": "env test", "locator":{"url": "https://radio-t.com", "site": "remark"}}`))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
// wait for webhook to be sent before shutting down
|
||||
assert.Eventually(t, func() bool {
|
||||
return webhookSent.Load() == int32(1)
|
||||
}, 30*time.Second, 10*time.Millisecond, "webhook was not sent")
|
||||
}
|
||||
|
||||
func TestGetDump(t *testing.T) {
|
||||
dump := getDump()
|
||||
assert.Contains(t, dump, "goroutine")
|
||||
assert.Contains(t, dump, "[running]")
|
||||
assert.Contains(t, dump, "backend/app/main.go")
|
||||
assert.True(t, strings.Contains(dump, "goroutine"))
|
||||
assert.True(t, strings.Contains(dump, "[running]"))
|
||||
assert.True(t, strings.Contains(dump, "backend/app/main.go"))
|
||||
t.Logf("\n dump: %s", dump)
|
||||
}
|
||||
|
||||
// chooseUnusedPort asks the kernel for a free port from the ephemeral range, which makes a
|
||||
// collision between concurrently running package test binaries very unlikely
|
||||
func chooseUnusedPort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", ":0")
|
||||
require.NoError(t, err, "no free port available")
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
func chooseRandomUnusedPort() (port int) {
|
||||
for i := 0; i < 10; i++ {
|
||||
port = 40000 + int(rand.Int31n(10000))
|
||||
if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
|
||||
_ = ln.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
// waitForHTTPServerStart blocks until the server on port answers, failing the test naming the
|
||||
// port if it never does
|
||||
func waitForHTTPServerStart(t *testing.T, port int) {
|
||||
t.Helper()
|
||||
func waitForHTTPServerStart(port int) {
|
||||
// wait for up to 10 seconds for server to start before returning it
|
||||
client := http.Client{Timeout: time.Second}
|
||||
defer client.CloseIdleConnections()
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
for i := 0; i < 100; i++ {
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}, 30*time.Second, 10*time.Millisecond, "http server on port %d didn't start", port)
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// both ignores are for leaks which are detected locally
|
||||
goleak.VerifyTestMain(
|
||||
m,
|
||||
// the shutdown goroutine in serverApp.run is not joined by Wait, and Rest.Shutdown gives
|
||||
// httpServer.Shutdown a second, which can outlast goleak's retry budget on a loaded runner
|
||||
goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
|
||||
goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
|
||||
// this will be fixed in https://github.com/hashicorp/golang-lru/issues/159
|
||||
goleak.IgnoreTopFunction("github.com/hashicorp/golang-lru/v2/expirable.NewLRU[...].func1"),
|
||||
// regexp2, pulled in by chroma for syntax highlighting, keeps one shared clock goroutine
|
||||
// alive for up to a second after the last match with a timeout, sleeping in 100ms ticks.
|
||||
// it ends on its own, but a binary that finishes inside that window is reported as leaking
|
||||
goleak.IgnoreAnyFunction("github.com/dlclark/regexp2/v2.runClock"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// AutoBackup struct handles daily backups params for siteID
|
||||
@@ -25,7 +27,6 @@ type AutoBackup struct {
|
||||
func (ab AutoBackup) Do(ctx context.Context) {
|
||||
log.Printf("[INFO] activate auto-backup for %s under %s, duration %s", ab.SiteID, ab.BackupLocation, ab.Duration)
|
||||
tick := time.NewTicker(ab.Duration)
|
||||
defer tick.Stop()
|
||||
log.Printf("[DEBUG] first backup for %s at %s", ab.SiteID, time.Now().Add(ab.Duration))
|
||||
|
||||
for {
|
||||
@@ -47,43 +48,38 @@ func (ab AutoBackup) Do(ctx context.Context) {
|
||||
func (ab AutoBackup) makeBackup() (string, error) {
|
||||
log.Printf("[DEBUG] make backup for %s", ab.SiteID)
|
||||
backupFile := fmt.Sprintf("%s/backup-%s-%s.gz", ab.BackupLocation, ab.SiteID, time.Now().Format("20060102"))
|
||||
fh, err := os.Create(backupFile) //nolint:gosec // harmless
|
||||
fh, err := os.Create(backupFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("can't create backup file %s: %w", backupFile, err)
|
||||
return "", errors.Wrapf(err, "can't create backup file %s", backupFile)
|
||||
}
|
||||
gz := gzip.NewWriter(fh)
|
||||
|
||||
if _, err = ab.Exporter.Export(gz, ab.SiteID); err != nil {
|
||||
return "", fmt.Errorf("export failed for %s: %w", ab.SiteID, err)
|
||||
return "", errors.Wrapf(err, "export failed for %s", ab.SiteID)
|
||||
}
|
||||
if err = gz.Close(); err != nil {
|
||||
return "", fmt.Errorf("can't close gz for %s: %w", backupFile, err)
|
||||
return "", errors.Wrapf(err, "can't close gz for %s", backupFile)
|
||||
}
|
||||
if err = fh.Close(); err != nil {
|
||||
return "", fmt.Errorf("can't close file handler for %s: %w", backupFile, err)
|
||||
return "", errors.Wrapf(err, "can't close file handler for %s", backupFile)
|
||||
}
|
||||
log.Printf("[DEBUG] created backup file %s", backupFile)
|
||||
return backupFile, nil
|
||||
}
|
||||
|
||||
func (ab AutoBackup) removeOldBackupFiles() {
|
||||
files, err := os.ReadDir(ab.BackupLocation)
|
||||
files, err := ioutil.ReadDir(ab.BackupLocation)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] can't read files in backup directory %s, %s", ab.BackupLocation, err)
|
||||
return
|
||||
}
|
||||
backFiles := []os.FileInfo{}
|
||||
for _, file := range files {
|
||||
info, e := file.Info()
|
||||
if e != nil {
|
||||
log.Printf("[WARN] can't read info for directory %s, %s", file.Name(), e)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(file.Name(), "backup-"+ab.SiteID) {
|
||||
backFiles = append(backFiles, info)
|
||||
backFiles = append(backFiles, file)
|
||||
}
|
||||
}
|
||||
sort.Slice(backFiles, func(i, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
|
||||
sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
|
||||
|
||||
if len(backFiles) > ab.KeepMax {
|
||||
for i := 0; i < len(backFiles)-ab.KeepMax; i++ {
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -18,20 +17,20 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
|
||||
loc := "/tmp/remark-backups.test"
|
||||
defer os.RemoveAll(loc)
|
||||
|
||||
assert.NoError(t, os.MkdirAll(loc, 0o700))
|
||||
assert.NoError(t, os.MkdirAll(loc, 0700))
|
||||
|
||||
for i := 1; i <= 10; i++ {
|
||||
fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
|
||||
err := os.WriteFile(fname, []byte("blah"), 0o600)
|
||||
err := ioutil.WriteFile(fname, []byte("blah"), 0600)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
|
||||
err := os.WriteFile(fname, []byte("blah"), 0o600)
|
||||
err := ioutil.WriteFile(fname, []byte("blah"), 0600)
|
||||
assert.NoError(t, err)
|
||||
|
||||
bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
|
||||
bk.removeOldBackupFiles()
|
||||
ff, err := os.ReadDir(loc)
|
||||
ff, err := ioutil.ReadDir(loc)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
|
||||
assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
|
||||
@@ -43,7 +42,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
|
||||
func TestBackup_MakeBackup(t *testing.T) {
|
||||
loc := "/tmp/remark-backups.test"
|
||||
defer os.RemoveAll(loc)
|
||||
assert.NoError(t, os.MkdirAll(loc, 0o700))
|
||||
assert.NoError(t, os.MkdirAll(loc, 0700))
|
||||
|
||||
bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
|
||||
fname, err := bk.makeBackup()
|
||||
@@ -51,50 +50,34 @@ func TestBackup_MakeBackup(t *testing.T) {
|
||||
expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
|
||||
assert.Equal(t, expFile, fname)
|
||||
|
||||
assert.Equal(t, exportedPayload, gzContent(t, expFile))
|
||||
fi, err := os.Lstat(expFile)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(52), fi.Size())
|
||||
}
|
||||
|
||||
func TestBackup_Do(t *testing.T) {
|
||||
loc := "/tmp/remark-backups.test"
|
||||
defer os.RemoveAll(loc)
|
||||
assert.NoError(t, os.MkdirAll(loc, 0o700))
|
||||
assert.NoError(t, os.MkdirAll(loc, 0700))
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(time.Second)
|
||||
cancel()
|
||||
}()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(time.Second)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}, Duration: 600 * time.Millisecond}
|
||||
bk.Do(ctx)
|
||||
bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}, Duration: 600 * time.Millisecond}
|
||||
bk.Do(ctx)
|
||||
|
||||
expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
|
||||
assert.Equal(t, exportedPayload, gzContent(t, expFile))
|
||||
})
|
||||
}
|
||||
|
||||
const exportedPayload = "some export blah blah 1234567890"
|
||||
|
||||
// the compressed size is not assertable: it moves with the compress/flate version
|
||||
func gzContent(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
fh, err := os.Open(name) //nolint:gosec // path is built by the test
|
||||
require.NoError(t, err)
|
||||
defer func() { assert.NoError(t, fh.Close()) }()
|
||||
|
||||
gz, err := gzip.NewReader(fh)
|
||||
require.NoError(t, err)
|
||||
defer func() { assert.NoError(t, gz.Close()) }()
|
||||
|
||||
b, err := io.ReadAll(gz)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
|
||||
fi, err := os.Lstat(expFile)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(52), fi.Size())
|
||||
}
|
||||
|
||||
type mockExporter struct{}
|
||||
|
||||
func (mock *mockExporter) Export(w io.Writer, _ string) (int, error) {
|
||||
_, err := w.Write([]byte(exportedPayload))
|
||||
_, err := w.Write([]byte("some export blah blah 1234567890"))
|
||||
return 1000, err
|
||||
}
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
)
|
||||
|
||||
// Commento implements Importer from commento export json
|
||||
type Commento struct {
|
||||
DataStore Store
|
||||
}
|
||||
|
||||
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/domain_import_commento.go#L11-L15
|
||||
type commentoExport struct {
|
||||
Version int `json:"version"`
|
||||
Comments []commentoComment `json:"comments"`
|
||||
Commenters []commentoCommenter `json:"commenters"`
|
||||
}
|
||||
|
||||
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/comment.go#L7-L20
|
||||
type commentoComment struct {
|
||||
CommentHex string `json:"commentHex"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Path string `json:"url,omitempty"`
|
||||
CommenterHex string `json:"commenterHex"`
|
||||
Markdown string `json:"markdown"`
|
||||
HTML string `json:"html"`
|
||||
ParentHex string `json:"parentHex"`
|
||||
Score int `json:"score"`
|
||||
State string `json:"state,omitempty"`
|
||||
CreationDate time.Time `json:"creationDate"`
|
||||
Direction int `json:"direction"`
|
||||
Deleted bool `json:"deleted"`
|
||||
}
|
||||
|
||||
// Credit: https://gitlab.com/commento/commento/-/blob/master/api/commenter.go#L7-L16
|
||||
type commentoCommenter struct {
|
||||
CommenterHex string `json:"commenterHex,omitempty"`
|
||||
Email string `json:"email,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Link string `json:"link"`
|
||||
Photo string `json:"photo"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
JoinDate time.Time `json:"joinDate"`
|
||||
IsModerator bool `json:"isModerator"`
|
||||
}
|
||||
|
||||
// Import comments from Commento and save to store
|
||||
func (d *Commento) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
if e := d.DataStore.DeleteAll(siteID); e != nil {
|
||||
return 0, e
|
||||
}
|
||||
|
||||
commentsCh := d.convert(r, siteID)
|
||||
failed, passed := 0, 0
|
||||
for c := range commentsCh {
|
||||
if _, err = d.DataStore.Create(c); err != nil {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
passed++
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
err = fmt.Errorf("failed to save %d comments", failed)
|
||||
if passed == 0 {
|
||||
err = fmt.Errorf("import failed")
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
|
||||
|
||||
return passed, err
|
||||
}
|
||||
|
||||
func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
commentsCh := make(chan store.Comment)
|
||||
|
||||
decoder := json.NewDecoder(r)
|
||||
|
||||
go func() {
|
||||
var exportedData commentoExport
|
||||
err := decoder.Decode(&exportedData)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] can't decode commento export json, %s", err.Error())
|
||||
}
|
||||
|
||||
usersMap := map[string]store.User{}
|
||||
for _, commenter := range exportedData.Commenters {
|
||||
usersMap[commenter.CommenterHex] = store.User{
|
||||
Name: commenter.Name,
|
||||
ID: "commento_" + store.EncodeID(commenter.CommenterHex),
|
||||
Picture: commenter.Photo,
|
||||
}
|
||||
}
|
||||
|
||||
usersMap["anonymous"] = store.User{
|
||||
Name: "Anonymous",
|
||||
ID: "commento_" + store.EncodeID("anonymous"),
|
||||
}
|
||||
|
||||
for _, comment := range exportedData.Comments {
|
||||
u, ok := usersMap[comment.CommenterHex]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if comment.Deleted {
|
||||
continue
|
||||
}
|
||||
|
||||
parentID := comment.ParentHex
|
||||
// comments with ParentHex == "root" are top-level comments
|
||||
if parentID == "root" {
|
||||
parentID = ""
|
||||
}
|
||||
|
||||
commentURL, e := url.JoinPath("https://", comment.Domain, comment.Path)
|
||||
if e != nil {
|
||||
log.Printf("[WARN] can't construct comment URL in commento import, %s", err.Error())
|
||||
}
|
||||
log.Printf("[ERROR] commentoURL: %s", commentURL)
|
||||
|
||||
c := store.Comment{
|
||||
ID: comment.CommentHex,
|
||||
Locator: store.Locator{
|
||||
URL: commentURL,
|
||||
SiteID: siteID,
|
||||
},
|
||||
User: u,
|
||||
Text: comment.Markdown,
|
||||
Timestamp: comment.CreationDate,
|
||||
ParentID: parentID,
|
||||
Imported: true,
|
||||
}
|
||||
|
||||
commentsCh <- c
|
||||
}
|
||||
|
||||
close(commentsCh)
|
||||
}()
|
||||
|
||||
return commentsCh
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestCommento_Import(t *testing.T) {
|
||||
defer os.Remove("/tmp/remark-test.db")
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
require.NoError(t, err, "create store")
|
||||
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
|
||||
d := Commento{DataStore: &dataStore}
|
||||
fh, err := os.Open("testdata/commento.json")
|
||||
require.NoError(t, err)
|
||||
size, err := d.Import(fh, "test")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, size)
|
||||
|
||||
last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t, 3, len(last), "3 comments imported")
|
||||
|
||||
t.Log(last[0])
|
||||
|
||||
c := last[0] // last reverses, get first one
|
||||
assert.Equal(t, "Great reply!", c.Text)
|
||||
assert.Equal(t, "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9", c.ID)
|
||||
assert.Equal(t, "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854", c.ParentID)
|
||||
assert.Equal(t, store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"}, c.Locator)
|
||||
assert.Equal(t, "Saturnin Uf", c.User.Name)
|
||||
assert.Equal(t, "commento_35369aeb6ac5255de30410a0f86dc71eb9c6d0ca", c.User.ID)
|
||||
assert.True(t, c.Imported)
|
||||
|
||||
c = last[2] // anonymous comment
|
||||
assert.Equal(t, "Example comment created by user.", c.Text)
|
||||
assert.Equal(t, "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945", c.ID)
|
||||
assert.Equal(t, "", c.ParentID)
|
||||
assert.Equal(t, store.Locator{SiteID: "test", URL: "https://example.com/blog/post/2"}, c.Locator)
|
||||
assert.Equal(t, "Anonymous", c.User.Name)
|
||||
assert.Equal(t, "commento_0a92fab3230134cca6eadd9898325b9b2ae67998", c.User.ID)
|
||||
assert.True(t, c.Imported)
|
||||
|
||||
posts, err := dataStore.List("test", 0, 0)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(posts), "2 posts")
|
||||
|
||||
count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://example.com/blog/post/1"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, count)
|
||||
|
||||
count, err = dataStore.Count(store.Locator{SiteID: "test", URL: "https://example.com/blog/post/2"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
@@ -2,14 +2,14 @@ package migrator
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
// Disqus implements Importer from disqus xml
|
||||
@@ -44,7 +44,6 @@ type disqusComment struct {
|
||||
Tid uid `xml:"thread"`
|
||||
Pid uid `xml:"parent"`
|
||||
IsSpam bool `xml:"isSpam"`
|
||||
Deleted bool `xml:"isDeleted"`
|
||||
}
|
||||
|
||||
type uid struct {
|
||||
@@ -53,8 +52,9 @@ type uid struct {
|
||||
|
||||
// Import from disqus and save to store
|
||||
func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
if e := d.DataStore.DeleteAll(siteID); e != nil {
|
||||
return 0, e
|
||||
|
||||
if err = d.DataStore.DeleteAll(siteID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
commentsCh := d.convert(r, siteID)
|
||||
@@ -68,9 +68,9 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
err = fmt.Errorf("failed to save %d comments", failed)
|
||||
err = errors.Errorf("failed to save %d comments", failed)
|
||||
if passed == 0 {
|
||||
err = fmt.Errorf("import failed")
|
||||
err = errors.New("import failed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,15 +82,15 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
// convert disqus stream (xml) from reader and fill channel of comments.
|
||||
// runs async and closes channel on completion.
|
||||
func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
|
||||
postsMap := map[string]string{} // tid:url
|
||||
decoder := xml.NewDecoder(r)
|
||||
commentsCh := make(chan store.Comment)
|
||||
|
||||
stats := struct {
|
||||
inpThreads, inpComments int
|
||||
commentsCount, spamComments int
|
||||
failedThreads, failedPosts int
|
||||
deletedComments, skippedComments int
|
||||
inpThreads, inpComments int
|
||||
commentsCount, spamComments int
|
||||
failedThreads, failedPosts int
|
||||
}{}
|
||||
|
||||
go func() {
|
||||
@@ -100,7 +100,8 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
break
|
||||
}
|
||||
|
||||
if se, ok := t.(xml.StartElement); ok {
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
if se.Name.Local == "thread" {
|
||||
stats.inpThreads++
|
||||
thread := disqusThread{}
|
||||
@@ -109,13 +110,9 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
stats.failedThreads++
|
||||
continue
|
||||
}
|
||||
if thread.Deleted {
|
||||
continue
|
||||
}
|
||||
postsMap[thread.UID] = thread.Link
|
||||
continue
|
||||
}
|
||||
|
||||
if se.Name.Local == "post" {
|
||||
stats.inpComments++
|
||||
comment := disqusComment{}
|
||||
@@ -124,24 +121,13 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
stats.failedPosts++
|
||||
continue
|
||||
}
|
||||
if comment.Deleted {
|
||||
stats.deletedComments++
|
||||
continue
|
||||
}
|
||||
if comment.IsSpam {
|
||||
stats.spamComments++
|
||||
continue
|
||||
}
|
||||
|
||||
url, ok := postsMap[comment.Tid.Val]
|
||||
if !ok {
|
||||
stats.skippedComments++
|
||||
continue
|
||||
}
|
||||
|
||||
c := store.Comment{
|
||||
ID: comment.UID,
|
||||
Locator: store.Locator{URL: url, SiteID: siteID},
|
||||
Locator: store.Locator{URL: postsMap[comment.Tid.Val], SiteID: siteID},
|
||||
User: store.User{
|
||||
ID: "disqus_" + store.EncodeID(comment.AuthorUserName),
|
||||
Name: comment.AuthorName,
|
||||
@@ -150,10 +136,9 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
Text: d.cleanText(comment.Message),
|
||||
Timestamp: comment.CreatedAt,
|
||||
ParentID: comment.Pid.Val,
|
||||
Imported: true,
|
||||
}
|
||||
if comment.AuthorUserName == "" { // empty comment.AuthorUserName from disqus
|
||||
c.User.ID = "disqus_" + store.EncodeID(c.User.Name)
|
||||
if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
|
||||
c.User.ID = "disqus_" + c.User.Name
|
||||
}
|
||||
if c.ID == "" { // no comment.UID
|
||||
c.ID = comment.ID
|
||||
@@ -174,8 +159,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
|
||||
}
|
||||
|
||||
func (*Disqus) cleanText(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
text = strings.ReplaceAll(text, "\n", "")
|
||||
text = strings.ReplaceAll(text, "\t", "")
|
||||
text = strings.Replace(text, "\n", "", -1)
|
||||
text = strings.Replace(text, "\t", "", -1)
|
||||
return text
|
||||
}
|
||||
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestDisqus_Import(t *testing.T) {
|
||||
@@ -23,9 +23,7 @@ func TestDisqus_Import(t *testing.T) {
|
||||
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
d := Disqus{DataStore: &dataStore}
|
||||
fh, err := os.Open("testdata/disqus.xml")
|
||||
require.NoError(t, err)
|
||||
size, err := d.Import(fh, "test")
|
||||
size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 4, size)
|
||||
|
||||
@@ -41,11 +39,6 @@ func TestDisqus_Import(t *testing.T) {
|
||||
assert.Equal(t, "Alexander Blah", c.User.Name)
|
||||
assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID)
|
||||
assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
|
||||
assert.True(t, c.Imported)
|
||||
|
||||
c = last[1] // get comment with empty username
|
||||
assert.Equal(t, "No Username", c.User.Name)
|
||||
assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
|
||||
|
||||
posts, err := dataStore.List("test", 0, 0)
|
||||
assert.NoError(t, err)
|
||||
@@ -56,73 +49,11 @@ func TestDisqus_Import(t *testing.T) {
|
||||
assert.Equal(t, 2, count)
|
||||
}
|
||||
|
||||
func TestDisqus_ImportDeletedThread(t *testing.T) {
|
||||
defer os.Remove("/tmp/remark-test.db")
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
require.NoError(t, err, "create store")
|
||||
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
d := Disqus{DataStore: &dataStore}
|
||||
fh, err := os.Open("testdata/disqus-deleted-thread.xml")
|
||||
require.NoError(t, err)
|
||||
size, err := d.Import(fh, "test")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, size)
|
||||
|
||||
last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t, 2, len(last), "2 comments imported")
|
||||
|
||||
c := last[len(last)-1] // last reverses, get first one
|
||||
assert.True(t, strings.HasPrefix(c.Text, "<p>Google App Engine "), c.Text)
|
||||
assert.Equal(t, "299986072", c.ID)
|
||||
assert.Equal(t, "", c.ParentID)
|
||||
assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator)
|
||||
assert.Equal(t, "No Username", c.User.Name)
|
||||
assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
|
||||
assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP)
|
||||
assert.True(t, c.Imported)
|
||||
|
||||
c = last[1] // get comment with empty username
|
||||
assert.Equal(t, "No Username", c.User.Name)
|
||||
assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
|
||||
}
|
||||
|
||||
func TestDisqus_ImportDeletedPost(t *testing.T) {
|
||||
defer os.Remove("/tmp/remark-test.db")
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
require.NoError(t, err, "create store")
|
||||
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
d := Disqus{DataStore: &dataStore}
|
||||
fh, err := os.Open("testdata/disqus-deleted-post.xml")
|
||||
require.NoError(t, err)
|
||||
size, err := d.Import(fh, "test")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, size, "1 post deleted")
|
||||
|
||||
last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t, 3, len(last), "3 comments imported")
|
||||
|
||||
c := last[len(last)-1] // last reverses, get first one
|
||||
assert.True(t, strings.HasPrefix(c.Text, "<p>Microsoft "), c.Text)
|
||||
assert.Equal(t, "299744309", c.ID)
|
||||
assert.Equal(t, "", c.ParentID)
|
||||
assert.Equal(t, store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"}, c.Locator)
|
||||
assert.Equal(t, "mikhail", c.User.Name)
|
||||
assert.Equal(t, "disqus_1b6709749c0cab163db9070cc4edf3322b398d8c", c.User.ID)
|
||||
assert.Equal(t, "9d3657a95a4e341510404bd8bf1a363faefd4ba4", c.User.IP)
|
||||
assert.True(t, c.Imported)
|
||||
}
|
||||
|
||||
func TestDisqus_Convert(t *testing.T) {
|
||||
d := Disqus{}
|
||||
fh, err := os.Open("testdata/disqus.xml")
|
||||
require.NoError(t, err)
|
||||
ch := d.convert(fh, "test")
|
||||
ch := d.convert(strings.NewReader(xmlTestDisqus), "test")
|
||||
|
||||
res := make([]store.Comment, 0, 4)
|
||||
res := []store.Comment{}
|
||||
for comment := range ch {
|
||||
res = append(res, comment)
|
||||
}
|
||||
@@ -140,8 +71,160 @@ func TestDisqus_Convert(t *testing.T) {
|
||||
ID: "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434",
|
||||
IP: "178.178.178.178",
|
||||
},
|
||||
Imported: true,
|
||||
}
|
||||
exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z")
|
||||
assert.Equal(t, exp0, res[0])
|
||||
}
|
||||
|
||||
var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
|
||||
|
||||
<category dsq:id="707279">
|
||||
<forum>radiot</forum>
|
||||
<title>General</title>
|
||||
<isDefault>true</isDefault>
|
||||
</category>
|
||||
|
||||
<thread dsq:id="247918464">
|
||||
<id/>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T20:46:25Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>98.212.28.115</ipAddress>
|
||||
<isClosed>false</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
<thread dsq:id="247937687">
|
||||
<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T21:17:17Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>80.250.214.235</ipAddress>
|
||||
<isClosed>true</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
|
||||
<post dsq:id="299619020">
|
||||
<id>3565798471341011339</id>
|
||||
<message>
|
||||
<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T15:16:29Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email/>
|
||||
<name>Alexander Blah</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>facebook-1787732238</username>
|
||||
</author>
|
||||
<ipAddress>178.178.178.178</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299744309">
|
||||
<id>3029154520436241933</id>
|
||||
<message>
|
||||
<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T17:44:22Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>mihail.noname@gmail.com</email>
|
||||
<name>mikhail</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>mikhail-noname</username>
|
||||
</author>
|
||||
<ipAddress>195.195.195.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986072">
|
||||
<id>6580890074280459209</id>
|
||||
<message>
|
||||
<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>dmitri.noname@gmail.com</email>
|
||||
<name>Dmitry Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>google-74b9e7568ef6860e93862c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>89.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post>
|
||||
<id>12345678890</id>
|
||||
<message>This comment had no ID</message>
|
||||
<createdAt>2011-08-31T22:49:43Z</createdAt>
|
||||
<forum>radiot</forum>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>blah.noname@gmail.com</email>
|
||||
<name>Blah Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>74b9e7568ef6860e93862c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986073">
|
||||
<id>6580890074280459219</id>
|
||||
<message>some ugly spam</message>
|
||||
<createdAt>2011-09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>true</isSpam>
|
||||
<author>
|
||||
<email>spam.noname@gmail.com</email>
|
||||
<name>Spam Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="x299986073">
|
||||
<message>some bad comment</message>
|
||||
<createdAt>2011-x09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>123</isSpam>
|
||||
<author>
|
||||
<email>noname@gmail.com</email>
|
||||
<name>Noname</name>
|
||||
<isAnonymous>true</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.39</ipAddress>
|
||||
<thread dsq:id=247937687/>
|
||||
</post>
|
||||
|
||||
</disqus>
|
||||
`
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -29,7 +30,7 @@ func NewURLMapper(reader io.Reader) (Mapper, error) {
|
||||
// https://www.myblog.com/blog/1/ https://myblog.com/blog/1/
|
||||
// https://www.myblog.com/* https://myblog.com/*
|
||||
func (u *URLMapper) loadRules(reader io.Reader) error {
|
||||
data, err := io.ReadAll(reader)
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -38,11 +39,11 @@ func (u *URLMapper) loadRules(reader io.Reader) error {
|
||||
|
||||
u.rules = make(map[string]string)
|
||||
|
||||
for row := range strings.SplitSeq(rulesText, "\n") {
|
||||
for _, row := range strings.Split(rulesText, "\n") {
|
||||
row = strings.TrimSpace(row)
|
||||
urls := strings.Split(row, " ")
|
||||
if len(urls) != 2 {
|
||||
return fmt.Errorf("bad row %s", row)
|
||||
return errors.New("bad row " + row)
|
||||
}
|
||||
|
||||
from, to := strings.TrimSpace(urls[0]), strings.TrimSpace(urls[1])
|
||||
@@ -64,8 +65,8 @@ func (u *URLMapper) URL(url string) string {
|
||||
}
|
||||
oldURL = strings.TrimSuffix(oldURL, "*")
|
||||
newURL = strings.TrimSuffix(newURL, "*")
|
||||
if after, ok := strings.CutPrefix(url, oldURL); ok {
|
||||
return newURL + after
|
||||
if strings.HasPrefix(url, oldURL) {
|
||||
return newURL + strings.TrimPrefix(url, oldURL)
|
||||
}
|
||||
}
|
||||
// search failed, return given url
|
||||
|
||||
@@ -4,14 +4,14 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
// Importer defines interface to convert posts from external sources
|
||||
@@ -38,7 +38,7 @@ type MapperMaker func(reader io.Reader) (Mapper, error)
|
||||
type Store interface {
|
||||
Create(comment store.Comment) (commentID string, err error)
|
||||
Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
|
||||
List(siteID string, limit, skip int) ([]store.PostInfo, error)
|
||||
List(siteID string, limit int, skip int) ([]store.PostInfo, error)
|
||||
DeleteAll(siteID string) error
|
||||
Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
|
||||
SetMetas(siteID string, umetas []service.UserMetaData, pmetas []service.PostMetaData) error
|
||||
@@ -64,20 +64,18 @@ func ImportComments(p ImportParams) (int, error) {
|
||||
importer = &Disqus{DataStore: p.DataStore}
|
||||
case "wordpress":
|
||||
importer = &WordPress{DataStore: p.DataStore}
|
||||
case "commento":
|
||||
importer = &Commento{DataStore: p.DataStore}
|
||||
case "native":
|
||||
importer = &Native{DataStore: p.DataStore}
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported import provider %s", p.Provider)
|
||||
return 0, errors.Errorf("unsupported import provider %s", p.Provider)
|
||||
}
|
||||
|
||||
fh, err := os.Open(p.InputFile)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("can't open import file %s: %w", p.InputFile, err)
|
||||
return 0, errors.Wrapf(err, "can't open import file %s", p.InputFile)
|
||||
}
|
||||
|
||||
defer func() { //nolint:gosec // false positive on defer without error check when it's checked here
|
||||
defer func() {
|
||||
if err = fh.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close %s, %s", p.InputFile, err)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -9,14 +10,20 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestMigrator_ImportDisqus(t *testing.T) {
|
||||
defer os.Remove("/tmp/remark-test.db")
|
||||
defer func() {
|
||||
os.Remove("/tmp/remark-test.db")
|
||||
os.Remove("/tmp/disqus-test.xml")
|
||||
}()
|
||||
|
||||
err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
|
||||
require.NoError(t, err)
|
||||
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
require.NoError(t, err, "create store")
|
||||
@@ -24,7 +31,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
|
||||
defer dataStore.Close()
|
||||
size, err := ImportComments(ImportParams{
|
||||
DataStore: dataStore,
|
||||
InputFile: "testdata/disqus.xml",
|
||||
InputFile: "/tmp/disqus-test.xml",
|
||||
SiteID: "test",
|
||||
Provider: "disqus",
|
||||
})
|
||||
@@ -42,7 +49,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
|
||||
os.Remove("/tmp/wordpress-test.xml")
|
||||
}()
|
||||
|
||||
err := os.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0o600)
|
||||
err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
|
||||
require.NoError(t, err)
|
||||
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
@@ -63,27 +70,6 @@ func TestMigrator_ImportWordPress(t *testing.T) {
|
||||
assert.Equal(t, 3, len(last), "3 comments imported")
|
||||
}
|
||||
|
||||
func TestMigrator_ImportCommento(t *testing.T) {
|
||||
defer os.Remove("/tmp/remark-test.db")
|
||||
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
|
||||
require.NoError(t, err, "create store")
|
||||
dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
size, err := ImportComments(ImportParams{
|
||||
DataStore: dataStore,
|
||||
InputFile: "testdata/commento.json",
|
||||
SiteID: "test",
|
||||
Provider: "commento",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, size)
|
||||
|
||||
last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(last), "3 comments imported")
|
||||
}
|
||||
|
||||
func TestMigrator_ImportNative(t *testing.T) {
|
||||
defer func() {
|
||||
os.Remove("/tmp/remark-test.db")
|
||||
@@ -93,7 +79,7 @@ func TestMigrator_ImportNative(t *testing.T) {
|
||||
data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
|
||||
`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
|
||||
|
||||
err := os.WriteFile("/tmp/disqus-test.r42", []byte(data), 0o600)
|
||||
err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
|
||||
require.NoError(t, err)
|
||||
|
||||
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
|
||||
|
||||
@@ -4,16 +4,15 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/go-pkgz/syncs"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
const nativeVersion = 1
|
||||
@@ -36,8 +35,9 @@ type meta struct {
|
||||
// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
|
||||
// The final file is a valid json
|
||||
func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
|
||||
|
||||
if err = n.exportMeta(siteID, w); err != nil {
|
||||
return 0, fmt.Errorf("failed to export meta for site %s: %w", siteID, err)
|
||||
return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID)
|
||||
}
|
||||
|
||||
topics, err := n.DataStore.List(siteID, 0, 0)
|
||||
@@ -47,23 +47,24 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
|
||||
|
||||
log.Printf("[DEBUG] exporting %d topics", len(topics))
|
||||
commentsCount := 0
|
||||
for _, topic := range slices.Backward(topics) { // topics from List sorted in opposite direction
|
||||
|
||||
for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
|
||||
topic := topics[i]
|
||||
comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
|
||||
if e != nil {
|
||||
return commentsCount, e
|
||||
}
|
||||
|
||||
for _, comment := range comments {
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
enc := json.NewEncoder(buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
|
||||
if err = enc.Encode(comment); err != nil {
|
||||
return commentsCount, fmt.Errorf("can't marshal %v: %w", comments, err)
|
||||
return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
|
||||
}
|
||||
if _, err = w.Write(buf.Bytes()); err != nil {
|
||||
return commentsCount, fmt.Errorf("can't write comment data: %w", err)
|
||||
return commentsCount, errors.Wrap(err, "can't write comment data")
|
||||
}
|
||||
commentsCount++
|
||||
}
|
||||
@@ -77,11 +78,11 @@ func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
|
||||
m := meta{Version: nativeVersion}
|
||||
m.Users, m.Posts, err = n.DataStore.Metas(siteID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get meta: %w", err)
|
||||
return errors.Wrap(err, "can't get meta")
|
||||
}
|
||||
|
||||
if err = json.NewEncoder(w).Encode(m); err != nil {
|
||||
return fmt.Errorf("can't encode meta: %w", err)
|
||||
return errors.Wrap(err, "can't encode meta")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -132,15 +133,15 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
|
||||
m := meta{}
|
||||
dec := json.NewDecoder(reader)
|
||||
if err = dec.Decode(&m); err != nil {
|
||||
return 0, fmt.Errorf("failed to import meta for site %s: %w", siteID, err)
|
||||
return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
|
||||
}
|
||||
|
||||
if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
|
||||
return 0, fmt.Errorf("unexpected import file version %d", m.Version)
|
||||
return 0, errors.Errorf("unexpected import file version %d", m.Version)
|
||||
}
|
||||
|
||||
if e := n.DataStore.DeleteAll(siteID); e != nil {
|
||||
return 0, e
|
||||
if err = n.DataStore.DeleteAll(siteID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var failed, total, comments int64
|
||||
@@ -154,7 +155,6 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
|
||||
for {
|
||||
comment := store.Comment{}
|
||||
err = dec.Decode(&comment)
|
||||
comment.Imported = true
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
@@ -179,12 +179,13 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
|
||||
log.Printf("[DEBUG] imported %d comments", num)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
grp.Wait()
|
||||
|
||||
if failed > 0 {
|
||||
return int(comments), fmt.Errorf("failed to save %d comments", failed)
|
||||
return int(comments), errors.Errorf("failed to save %d comments", failed)
|
||||
}
|
||||
log.Printf("[INFO] imported %d comments from %d records", comments, total)
|
||||
|
||||
|
||||
@@ -14,10 +14,10 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestNative_Export(t *testing.T) {
|
||||
@@ -73,7 +73,7 @@ func TestNative_Import(t *testing.T) {
|
||||
|
||||
inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
|
||||
{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
|
||||
{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00","imported":false}`
|
||||
{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
|
||||
|
||||
b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
|
||||
r := Native{DataStore: b}
|
||||
@@ -87,12 +87,10 @@ func TestNative_Import(t *testing.T) {
|
||||
assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
|
||||
assert.Equal(t, "1234", comments[0].ParentID)
|
||||
assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
|
||||
assert.True(t, comments[0].Imported)
|
||||
|
||||
assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
|
||||
assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
|
||||
assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
|
||||
assert.True(t, comments[1].Imported)
|
||||
|
||||
assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
|
||||
assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
|
||||
@@ -153,6 +151,7 @@ func TestNative_ImportWrongVersion(t *testing.T) {
|
||||
size, err := r.Import(strings.NewReader(inp), "radio-t")
|
||||
assert.EqualError(t, err, "unexpected import file version 2")
|
||||
assert.Equal(t, 0, size)
|
||||
|
||||
}
|
||||
func TestNative_ImportManyWithError(t *testing.T) {
|
||||
b, teardown := prep(t) // write 2 comments
|
||||
@@ -162,8 +161,8 @@ func TestNative_ImportManyWithError(t *testing.T) {
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
|
||||
for i := range 100 {
|
||||
fmt.Fprintf(buf, goodRec, i)
|
||||
for i := 0; i < 100; i++ {
|
||||
buf.WriteString(fmt.Sprintf(goodRec, i))
|
||||
}
|
||||
buf.WriteString("{}\n")
|
||||
buf.WriteString("{}\n")
|
||||
@@ -179,10 +178,11 @@ func TestNative_ImportManyWithError(t *testing.T) {
|
||||
}
|
||||
|
||||
// makes new boltdb, put two records
|
||||
func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
|
||||
testDB := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
|
||||
func prep(t *testing.T) (*service.DataStore, func()) {
|
||||
|
||||
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDB})
|
||||
testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
|
||||
|
||||
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
|
||||
assert.NoError(t, err)
|
||||
|
||||
b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
@@ -207,6 +207,6 @@ func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
|
||||
|
||||
return b, func() {
|
||||
require.NoError(t, b.Close())
|
||||
_ = os.Remove(testDB)
|
||||
_ = os.Remove(testDb)
|
||||
}
|
||||
}
|
||||
|
||||
-69
@@ -1,69 +0,0 @@
|
||||
{
|
||||
"version": 1,
|
||||
"comments": [
|
||||
{
|
||||
"commentHex": "e7069a7dfcfaed43caf62300a9b0edb1c124ad79d0f5887c93649c15d7f69945",
|
||||
"domain": "example.com",
|
||||
"url": "/blog/post/2",
|
||||
"commenterHex": "anonymous",
|
||||
"markdown": "Example comment created by user.",
|
||||
"html": "",
|
||||
"parentHex": "root",
|
||||
"score": 1,
|
||||
"state": "approved",
|
||||
"creationDate": "2021-03-12T11:21:56Z",
|
||||
"direction": 0,
|
||||
"deleted": false
|
||||
},
|
||||
{
|
||||
"commentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
|
||||
"domain": "example.com",
|
||||
"url": "/blog/post/1",
|
||||
"commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
|
||||
"markdown": "Example 2 comment created by user.",
|
||||
"html": "",
|
||||
"parentHex": "root",
|
||||
"score": 0,
|
||||
"state": "approved",
|
||||
"creationDate": "2021-03-17T12:09:47.722181Z",
|
||||
"direction": 0,
|
||||
"deleted": false
|
||||
},
|
||||
{
|
||||
"commentHex": "ea5f7bcd6ac9bb7b657f7d0569831104e1bcf9c253d03c1e16bf9654c49a5ce9",
|
||||
"domain": "example.com",
|
||||
"url": "/blog/post/1",
|
||||
"commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
|
||||
"markdown": "Great reply!",
|
||||
"html": "",
|
||||
"parentHex": "7d77e39fcd813241d6281478cc8f21ab5f807d043c750bc1a936bc23b34fb854",
|
||||
"score": 0,
|
||||
"state": "approved",
|
||||
"creationDate": "2021-05-11T15:43:01.852651Z",
|
||||
"direction": 0,
|
||||
"deleted": false
|
||||
}
|
||||
],
|
||||
"commenters": [
|
||||
{
|
||||
"commenterHex": "a1ac58ed1146bd7fe3feff6a7276f73955c3bfd23cacee00e2e0a7a89b1a8c10",
|
||||
"email": "somegreatmail@gmail.com",
|
||||
"name": "User5276",
|
||||
"link": "https://example.com/profile/257",
|
||||
"photo": "https://secure.gravatar.com/avatar/8f279626d26175134b0d5c88648172f7",
|
||||
"provider": "sso:example.com",
|
||||
"joinDate": "2021-03-19T19:27:25.954285Z",
|
||||
"isModerator": false
|
||||
},
|
||||
{
|
||||
"commenterHex": "bd1290ab5c858cf2a05903c2a9a61fd63399c6635db38cc6597002195e22e061",
|
||||
"email": "moregreatmail@gmail.com",
|
||||
"name": "Saturnin Uf",
|
||||
"link": "https://example.com/profile/259",
|
||||
"photo": "https://secure.gravatar.com/avatar/6481228d190f0286a42bee9041f9b1a1",
|
||||
"provider": "sso:example.com",
|
||||
"joinDate": "2021-03-21T12:15:37.536035Z",
|
||||
"isModerator": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
`<?xml version="1.0" encoding="utf-8"?>
|
||||
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
|
||||
|
||||
<category dsq:id="707279">
|
||||
<forum>radiot</forum>
|
||||
<title>General</title>
|
||||
<isDefault>true</isDefault>
|
||||
</category>
|
||||
|
||||
<thread dsq:id="247918464">
|
||||
<id/>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T20:46:25Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>98.212.28.115</ipAddress>
|
||||
<isClosed>false</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
<thread dsq:id="247937687">
|
||||
<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T21:17:17Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>80.250.214.235</ipAddress>
|
||||
<isClosed>true</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
|
||||
<post dsq:id="299619020">
|
||||
<id>3565798471341011339</id>
|
||||
<message>
|
||||
<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T15:16:29Z</createdAt>
|
||||
<isDeleted>true</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email/>
|
||||
<name>Alexander Blah</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>facebook-1787732238</username>
|
||||
</author>
|
||||
<ipAddress>178.178.178.178</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299744309">
|
||||
<id>3029154520436241933</id>
|
||||
<message>
|
||||
<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T17:44:22Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>mihail.noname@gmail.com</email>
|
||||
<name>mikhail</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>mikhail-noname</username>
|
||||
</author>
|
||||
<ipAddress>195.195.195.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986072">
|
||||
<id>6580890074280459209</id>
|
||||
<message>
|
||||
<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>john.nousername@gmail.com</email>
|
||||
<name>No Username</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
</author>
|
||||
<ipAddress>89.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post>
|
||||
<id>12345678890</id>
|
||||
<message>This comment had no ID</message>
|
||||
<createdAt>2011-08-31T22:49:43Z</createdAt>
|
||||
<forum>radiot</forum>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>blah.noname@gmail.com</email>
|
||||
<name>Blah Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>74b9e7568ef6860e93862c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986073">
|
||||
<id>6580890074280459219</id>
|
||||
<message>some ugly spam</message>
|
||||
<createdAt>2011-09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>true</isSpam>
|
||||
<author>
|
||||
<email>spam.noname@gmail.com</email>
|
||||
<name>Spam Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="x299986073">
|
||||
<message>some bad comment</message>
|
||||
<createdAt>2011-x09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>123</isSpam>
|
||||
<author>
|
||||
<email>noname@gmail.com</email>
|
||||
<name>Noname</name>
|
||||
<isAnonymous>true</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.39</ipAddress>
|
||||
<thread dsq:id=247937687/>
|
||||
</post>
|
||||
|
||||
</disqus>
|
||||
@@ -1,150 +0,0 @@
|
||||
`<?xml version="1.0" encoding="utf-8"?>
|
||||
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
|
||||
|
||||
<category dsq:id="707279">
|
||||
<forum>radiot</forum>
|
||||
<title>General</title>
|
||||
<isDefault>true</isDefault>
|
||||
</category>
|
||||
|
||||
<thread dsq:id="247918464">
|
||||
<id/>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T20:46:25Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>98.212.28.115</ipAddress>
|
||||
<isClosed>false</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
<thread dsq:id="247937687">
|
||||
<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T21:17:17Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>80.250.214.235</ipAddress>
|
||||
<isClosed>true</isClosed>
|
||||
<isDeleted>true</isDeleted>
|
||||
</thread>
|
||||
|
||||
|
||||
<post dsq:id="299619020">
|
||||
<id>3565798471341011339</id>
|
||||
<message>
|
||||
<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T15:16:29Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email/>
|
||||
<name>Alexander Blah</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>facebook-1787732238</username>
|
||||
</author>
|
||||
<ipAddress>178.178.178.178</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299744309">
|
||||
<id>3029154520436241933</id>
|
||||
<message>
|
||||
<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T17:44:22Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>mihail.noname@gmail.com</email>
|
||||
<name>mikhail</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>mikhail-noname</username>
|
||||
</author>
|
||||
<ipAddress>195.195.195.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986072">
|
||||
<id>6580890074280459209</id>
|
||||
<message>
|
||||
<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>john.nousername@gmail.com</email>
|
||||
<name>No Username</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
</author>
|
||||
<ipAddress>89.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post>
|
||||
<id>12345678890</id>
|
||||
<message>This comment had no ID</message>
|
||||
<createdAt>2011-08-31T22:49:43Z</createdAt>
|
||||
<forum>radiot</forum>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>blah.noname@gmail.com</email>
|
||||
<name>Blah Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>74b9e7568ef6860e93862c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986073">
|
||||
<id>6580890074280459219</id>
|
||||
<message>some ugly spam</message>
|
||||
<createdAt>2011-09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>true</isSpam>
|
||||
<author>
|
||||
<email>spam.noname@gmail.com</email>
|
||||
<name>Spam Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="x299986073">
|
||||
<message>some bad comment</message>
|
||||
<createdAt>2011-x09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>123</isSpam>
|
||||
<author>
|
||||
<email>noname@gmail.com</email>
|
||||
<name>Noname</name>
|
||||
<isAnonymous>true</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.39</ipAddress>
|
||||
<thread dsq:id=247937687/>
|
||||
</post>
|
||||
|
||||
</disqus>
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
`<?xml version="1.0" encoding="utf-8"?>
|
||||
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
|
||||
|
||||
<category dsq:id="707279">
|
||||
<forum>radiot</forum>
|
||||
<title>General</title>
|
||||
<isDefault>true</isDefault>
|
||||
</category>
|
||||
|
||||
<thread dsq:id="247918464">
|
||||
<id/>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T20:46:25Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>98.212.28.115</ipAddress>
|
||||
<isClosed>false</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
<thread dsq:id="247937687">
|
||||
<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
|
||||
<forum>radiot</forum>
|
||||
<category dsq:id="707279"/>
|
||||
<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
|
||||
<title>Радио-Т: Радио-Т 229</title>
|
||||
<message/>
|
||||
<createdAt>2011-03-07T21:17:17Z</createdAt>
|
||||
<author>
|
||||
<email>umputun@gmail.com</email>
|
||||
<name>Umputun</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>umputun</username>
|
||||
</author>
|
||||
<ipAddress>80.250.214.235</ipAddress>
|
||||
<isClosed>true</isClosed>
|
||||
<isDeleted>false</isDeleted>
|
||||
</thread>
|
||||
|
||||
|
||||
<post dsq:id="299619020">
|
||||
<id>3565798471341011339</id>
|
||||
<message>
|
||||
<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T15:16:29Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email/>
|
||||
<name>Alexander Blah</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>facebook-1787732238</username>
|
||||
</author>
|
||||
<ipAddress>178.178.178.178</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299744309">
|
||||
<id>3029154520436241933</id>
|
||||
<message>
|
||||
<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T17:44:22Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>mihail.noname@gmail.com</email>
|
||||
<name>mikhail</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>mikhail-noname</username>
|
||||
</author>
|
||||
<ipAddress>195.195.195.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986072">
|
||||
<id>6580890074280459209</id>
|
||||
<message>
|
||||
<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
|
||||
</message>
|
||||
<createdAt>2011-08-31T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>john.nousername@gmail.com</email>
|
||||
<name>No Username</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
</author>
|
||||
<ipAddress>89.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post>
|
||||
<id>12345678890</id>
|
||||
<message>This comment had no ID</message>
|
||||
<createdAt>2011-08-31T22:49:43Z</createdAt>
|
||||
<forum>radiot</forum>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>false</isSpam>
|
||||
<author>
|
||||
<email>blah.noname@gmail.com</email>
|
||||
<name>Blah Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>74b9e7568ef6860e93862c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247918464"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="299986073">
|
||||
<id>6580890074280459219</id>
|
||||
<message>some ugly spam</message>
|
||||
<createdAt>2011-09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>true</isSpam>
|
||||
<author>
|
||||
<email>spam.noname@gmail.com</email>
|
||||
<name>Spam Noname</name>
|
||||
<isAnonymous>false</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.139</ipAddress>
|
||||
<thread dsq:id="247937687"/>
|
||||
</post>
|
||||
|
||||
<post dsq:id="x299986073">
|
||||
<message>some bad comment</message>
|
||||
<createdAt>2011-x09-30T22:48:43Z</createdAt>
|
||||
<isDeleted>false</isDeleted>
|
||||
<isSpam>123</isSpam>
|
||||
<author>
|
||||
<email>noname@gmail.com</email>
|
||||
<name>Noname</name>
|
||||
<isAnonymous>true</isAnonymous>
|
||||
<username>google-2c5d77590123</username>
|
||||
</author>
|
||||
<ipAddress>189.89.89.39</ipAddress>
|
||||
<thread dsq:id=247937687/>
|
||||
</post>
|
||||
|
||||
</disqus>
|
||||
@@ -2,22 +2,21 @@ package migrator
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
const wpTimeLayout = "2006-01-02 15:04:05"
|
||||
|
||||
// WordPress implements Importer from WP xml
|
||||
type WordPress struct {
|
||||
DataStore Store
|
||||
DisableFancyTextFormatting bool
|
||||
DataStore Store
|
||||
}
|
||||
|
||||
type wpItem struct {
|
||||
@@ -61,8 +60,9 @@ func (w *WordPress) Convert(text string) string {
|
||||
|
||||
// Import comments from WP and save to store
|
||||
func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
if e := w.DataStore.DeleteAll(siteID); e != nil {
|
||||
return 0, e
|
||||
|
||||
if err = w.DataStore.DeleteAll(siteID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
commentsCh := w.convert(r, siteID)
|
||||
@@ -76,9 +76,9 @@ func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
err = fmt.Errorf("failed to save %d comments", failed)
|
||||
err = errors.Errorf("failed to save %d comments", failed)
|
||||
if passed == 0 {
|
||||
err = fmt.Errorf("import failed")
|
||||
err = errors.New("import failed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
|
||||
}
|
||||
|
||||
func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
|
||||
|
||||
decoder := xml.NewDecoder(r)
|
||||
commentsCh := make(chan store.Comment)
|
||||
|
||||
@@ -106,7 +107,8 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
|
||||
break
|
||||
}
|
||||
|
||||
if el, ok := t.(xml.StartElement); ok {
|
||||
switch el := t.(type) {
|
||||
case xml.StartElement:
|
||||
if el.Name.Local == "item" {
|
||||
stats.inpItems++
|
||||
item := wpItem{}
|
||||
@@ -137,9 +139,8 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
|
||||
Text: comment.Content,
|
||||
Timestamp: comment.Date.time,
|
||||
ParentID: comment.PID,
|
||||
Imported: true,
|
||||
}
|
||||
commentsCh <- commentFormatter.Format(c, w.DisableFancyTextFormatting)
|
||||
commentsCh <- commentFormatter.Format(c)
|
||||
stats.inpComments++
|
||||
if stats.inpComments%1000 == 0 {
|
||||
log.Printf("[DEBUG] processed %d comments", stats.inpComments)
|
||||
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark42/backend/app/store/admin"
|
||||
"github.com/umputun/remark42/backend/app/store/engine"
|
||||
"github.com/umputun/remark42/backend/app/store/service"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store/admin"
|
||||
"github.com/umputun/remark/backend/app/store/engine"
|
||||
"github.com/umputun/remark/backend/app/store/service"
|
||||
)
|
||||
|
||||
func TestWordPress_Import(t *testing.T) {
|
||||
@@ -24,7 +24,7 @@ func TestWordPress_Import(t *testing.T) {
|
||||
|
||||
dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
|
||||
defer dataStore.Close()
|
||||
wp := WordPress{DataStore: &dataStore, DisableFancyTextFormatting: false}
|
||||
wp := WordPress{DataStore: &dataStore}
|
||||
size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, size)
|
||||
@@ -41,8 +41,7 @@ func TestWordPress_Import(t *testing.T) {
|
||||
assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
|
||||
ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
|
||||
assert.Equal(t, ts, c.Timestamp)
|
||||
assert.Equal(t, "<p>«Mekkatorque» was over in that tent up to the right</p>\n", c.Text)
|
||||
assert.True(t, c.Imported)
|
||||
assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
|
||||
|
||||
posts, err := dataStore.List(siteID, 0, 0)
|
||||
assert.NoError(t, err)
|
||||
@@ -54,25 +53,13 @@ func TestWordPress_Import(t *testing.T) {
|
||||
count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, count)
|
||||
|
||||
// test with DisableFancyTextFormatting
|
||||
wp = WordPress{DataStore: &dataStore, DisableFancyTextFormatting: true}
|
||||
size, err = wp.Import(strings.NewReader(xmlTestWP), siteID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, size)
|
||||
|
||||
last, err = dataStore.Last(siteID, 10, time.Time{}, adminUser)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t, 3, len(last), "3 comments imported")
|
||||
|
||||
assert.Equal(t, "<p>"Mekkatorque" was over in that tent up to the right</p>\n", last[0].Text)
|
||||
}
|
||||
|
||||
func TestWordPress_Convert(t *testing.T) {
|
||||
wp := WordPress{}
|
||||
ch := wp.convert(strings.NewReader(xmlTestWP), "testWP")
|
||||
|
||||
comments := make([]store.Comment, 0, 3)
|
||||
comments := []store.Comment{}
|
||||
for c := range ch {
|
||||
comments = append(comments, c)
|
||||
}
|
||||
@@ -90,7 +77,6 @@ func TestWordPress_Convert(t *testing.T) {
|
||||
ID: "wordpress_" + store.EncodeID("Wednesday Reading « Cynwise's Battlefield Manual"),
|
||||
IP: "74.200.244.101",
|
||||
},
|
||||
Imported: true,
|
||||
}
|
||||
exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
|
||||
assert.Equal(t, exp1, comments[1])
|
||||
@@ -100,7 +86,7 @@ func TestWP_Convert_MD(t *testing.T) {
|
||||
wp := WordPress{}
|
||||
ch := wp.convert(strings.NewReader(xmlTestWPmd), "siteID")
|
||||
|
||||
comments := make([]store.Comment, 0, 3)
|
||||
comments := []store.Comment{}
|
||||
for c := range ch {
|
||||
comments = append(comments, c)
|
||||
}
|
||||
@@ -259,7 +245,7 @@ var xmlTestWP = `
|
||||
<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
|
||||
<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
|
||||
<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
|
||||
<wp:comment_content><![CDATA["Mekkatorque" was over in that tent up to the right]]></wp:comment_content>
|
||||
<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
|
||||
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
|
||||
<wp:comment_type><![CDATA[]]></wp:comment_type>
|
||||
<wp:comment_parent>13</wp:comment_parent>
|
||||
|
||||
+340
-174
@@ -3,52 +3,86 @@ package notify
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/url"
|
||||
"io"
|
||||
"mime/quotedprintable"
|
||||
"net"
|
||||
"net/smtp"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/go-pkgz/repeater/v2"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/templates"
|
||||
"github.com/go-pkgz/repeater"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// EmailParams contain settings for email notifications
|
||||
type EmailParams struct {
|
||||
From string // from email address
|
||||
AdminEmails []string // administrator emails to send copy of comment notification to
|
||||
MsgTemplatePath string // path to request message template
|
||||
VerificationSubject string // verification message sub
|
||||
VerificationTemplatePath string // path to verification template
|
||||
SubscribeURL string // full subscribe handler URL
|
||||
UnsubscribeURL string // full unsubscribe handler URL
|
||||
From string // from email address
|
||||
MsgTemplate string // request message template
|
||||
VerificationSubject string // verification message subject
|
||||
VerificationTemplate string // verification message template
|
||||
SubscribeURL string // full subscribe handler URL
|
||||
UnsubscribeURL string // full unsubscribe handler URL
|
||||
|
||||
TokenGenFn func(userID, email, site string) (string, error) // unsubscribe token generation function
|
||||
TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
|
||||
}
|
||||
|
||||
// SMTPParams contain settings for smtp server connection
|
||||
type SMTPParams struct {
|
||||
Host string // SMTP host
|
||||
Port int // SMTP port
|
||||
TLS bool // TLS auth
|
||||
Username string // user name
|
||||
Password string // password
|
||||
TimeOut time.Duration // TCP connection timeout
|
||||
}
|
||||
|
||||
// Email implements notify.Destination for email
|
||||
type Email struct {
|
||||
*ntf.Email
|
||||
|
||||
EmailParams
|
||||
SMTPParams
|
||||
|
||||
smtp smtpClientCreator
|
||||
msgTmpl *template.Template // parsed request message template
|
||||
verifyTmpl *template.Template // parsed verification message template
|
||||
}
|
||||
|
||||
// default email client implementation
|
||||
type emailClient struct{ smtpClientCreator }
|
||||
|
||||
// smtpClient interface defines subset of net/smtp used by email client
|
||||
type smtpClient interface {
|
||||
Mail(string) error
|
||||
Auth(smtp.Auth) error
|
||||
Rcpt(string) error
|
||||
Data() (io.WriteCloser, error)
|
||||
Quit() error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// smtpClientCreator interface defines function for creating new smtpClients
|
||||
type smtpClientCreator interface {
|
||||
Create(SMTPParams) (smtpClient, error)
|
||||
}
|
||||
|
||||
type emailMessage struct {
|
||||
from string
|
||||
to string
|
||||
message string
|
||||
}
|
||||
|
||||
// msgTmplData store data for message from request template execution
|
||||
type msgTmplData struct {
|
||||
UserName string
|
||||
UserPicture string
|
||||
CommentText template.HTML
|
||||
CommentText string
|
||||
CommentLink string
|
||||
CommentDate time.Time
|
||||
ParentUserName string
|
||||
ParentUserPicture string
|
||||
ParentCommentText template.HTML
|
||||
ParentCommentText string
|
||||
ParentCommentLink string
|
||||
ParentCommentDate time.Time
|
||||
PostTitle string
|
||||
@@ -57,30 +91,6 @@ type msgTmplData struct {
|
||||
ForAdmin bool
|
||||
}
|
||||
|
||||
// emailCommentPolicy sanitizes comment HTML for inclusion in notification emails.
|
||||
// It is intentionally stricter than the store-level UGC policy used for web rendering:
|
||||
// links (<a>) and images (<img>) are dropped so a comment can't smuggle phishing links
|
||||
// or remote tracking pixels into an email sent from the legitimate remark42 address,
|
||||
// while basic inline and block text formatting is preserved.
|
||||
var emailCommentPolicy = func() *bluemonday.Policy {
|
||||
p := bluemonday.NewPolicy()
|
||||
p.AllowElements(
|
||||
"p", "br", "hr", "div", "span",
|
||||
"b", "strong", "i", "em", "u", "s", "strike", "del", "ins", "sub", "sup", "mark", "small",
|
||||
"blockquote", "q", "cite",
|
||||
"code", "pre", "kbd", "samp", "var",
|
||||
"ul", "ol", "li", "dl", "dt", "dd",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6",
|
||||
)
|
||||
return p
|
||||
}()
|
||||
|
||||
// emailSafeHTML strips links and images from pre-rendered comment HTML and returns
|
||||
// it as template.HTML so html/template renders the remaining safe formatting as-is.
|
||||
func emailSafeHTML(commentHTML string) template.HTML {
|
||||
return template.HTML(emailCommentPolicy.Sanitize(commentHTML)) //nolint:gosec // sanitized above: <a>/<img> dropped, only formatting tags survive
|
||||
}
|
||||
|
||||
// verifyTmplData store data for verification message template execution
|
||||
type verifyTmplData struct {
|
||||
User string
|
||||
@@ -91,153 +101,186 @@ type verifyTmplData struct {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultVerificationSubject = "Email verification"
|
||||
defaultEmailTimeout = 10 * time.Second
|
||||
defaultEmailTemplatePath = "email_reply.html.tmpl"
|
||||
defaultEmailVerificationTemplatePath = "email_confirmation_subscription.html.tmpl"
|
||||
defaultVerificationSubject = "Email verification"
|
||||
defaultEmailTimeout = 10 * time.Second
|
||||
defaultEmailTemplate = `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width" />
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<style type="text/css">
|
||||
img {
|
||||
max-width: 100%;
|
||||
max-height: 250px;
|
||||
margin: 5px 0;
|
||||
display: block;
|
||||
color: #000;
|
||||
}
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: #0aa;
|
||||
}
|
||||
p {
|
||||
margin: 0 0 12px;
|
||||
}
|
||||
blockquote {
|
||||
margin: 10px 0;
|
||||
padding: 12px 12px 1px 12px;
|
||||
background: rgba(255,255,255,.5)
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
|
||||
<body>
|
||||
<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
|
||||
<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
|
||||
{{- if .ForAdmin}}
|
||||
<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
|
||||
{{- else }}
|
||||
<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
|
||||
{{- end }}
|
||||
<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
|
||||
{{- if .ParentCommentText}}
|
||||
<div style="margin-bottom: 12px; line-height: 24px; word-break: break-all;">
|
||||
<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; display: inline; vertical-align: middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
|
||||
<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
|
||||
<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
|
||||
<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
|
||||
</div>
|
||||
<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
|
||||
{{.ParentCommentText}}
|
||||
</div>
|
||||
{{- end }}
|
||||
<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px;">
|
||||
<div style="margin-bottom: 12px;" line-height: 24px;word-break: break-all;>
|
||||
<img src="{{.UserPicture}}" style="width: 24px; height: 24px; display:inline; vertical-align:middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
|
||||
<span style="font-size: 14px; font-weight: bold; color: #777">{{.UserName}}</span>
|
||||
<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.CommentDate.Format "02.01.2006 at 15:04"}}</span>
|
||||
<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
|
||||
</div>
|
||||
<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">{{.CommentText}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="text-align: center; font-size: 14px; margin-top: 32px;">
|
||||
<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a>{{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}</i>
|
||||
<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;"></div>
|
||||
{{- if .UnsubscribeLink}}
|
||||
<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
|
||||
{{- end }}
|
||||
<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
|
||||
<div style="opacity: 0;font-size: 1;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
defaultEmailVerificationTemplate = `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width" />
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
</head>
|
||||
<body>
|
||||
<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
|
||||
<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
|
||||
<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
|
||||
<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
|
||||
{{- if .SubscribeURL}}
|
||||
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
|
||||
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
|
||||
{{- end }}
|
||||
<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
|
||||
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
|
||||
<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into “token” field on comments page</i></p>
|
||||
<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
|
||||
</div>
|
||||
<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
)
|
||||
|
||||
// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
|
||||
func NewEmail(emailParams EmailParams, smtpParams ntf.SMTPParams) (*Email, error) {
|
||||
func NewEmail(emailParams EmailParams, smtpParams SMTPParams) (*Email, error) {
|
||||
// set up Email emailParams
|
||||
if smtpParams.TimeOut <= 0 {
|
||||
smtpParams.TimeOut = defaultEmailTimeout
|
||||
res := Email{EmailParams: emailParams}
|
||||
if res.MsgTemplate == "" {
|
||||
res.MsgTemplate = defaultEmailTemplate
|
||||
}
|
||||
if res.VerificationTemplate == "" {
|
||||
res.VerificationTemplate = defaultEmailVerificationTemplate
|
||||
}
|
||||
|
||||
res := Email{Email: ntf.NewEmail(smtpParams), EmailParams: emailParams}
|
||||
|
||||
if res.VerificationSubject == "" {
|
||||
res.VerificationSubject = defaultVerificationSubject
|
||||
}
|
||||
|
||||
// initialize templates
|
||||
err := res.setTemplates()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't set templates: %w", err)
|
||||
// set up SMTP emailParams
|
||||
res.smtp = &emailClient{}
|
||||
res.SMTPParams = smtpParams
|
||||
if res.TimeOut <= 0 {
|
||||
res.TimeOut = defaultEmailTimeout
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
|
||||
res.Host, res.Username, res.TimeOut)
|
||||
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func (e *Email) setTemplates() error {
|
||||
// initialize templates
|
||||
var err error
|
||||
var msgTmplFile, verifyTmplFile []byte
|
||||
|
||||
if e.VerificationTemplatePath == "" {
|
||||
e.VerificationTemplatePath = defaultEmailVerificationTemplatePath
|
||||
if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
|
||||
return nil, errors.Wrapf(err, "can't parse message template")
|
||||
}
|
||||
|
||||
if e.MsgTemplatePath == "" {
|
||||
e.MsgTemplatePath = defaultEmailTemplatePath
|
||||
if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
|
||||
return nil, errors.Wrapf(err, "can't parse verification template")
|
||||
}
|
||||
|
||||
if msgTmplFile, err = templates.Read(e.MsgTemplatePath); err != nil {
|
||||
return fmt.Errorf("can't read message template: %w", err)
|
||||
}
|
||||
if verifyTmplFile, err = templates.Read(e.VerificationTemplatePath); err != nil {
|
||||
return fmt.Errorf("can't read verification template: %w", err)
|
||||
}
|
||||
if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil {
|
||||
return fmt.Errorf("can't parse message template: %w", err)
|
||||
}
|
||||
if e.verifyTmpl, err = template.New("verifyTmpl").Parse(string(verifyTmplFile)); err != nil {
|
||||
return fmt.Errorf("can't parse verification template: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return &res, err
|
||||
}
|
||||
|
||||
// Send email about comment reply to Request.Emails and Email.AdminEmails
|
||||
// if they're set.
|
||||
// Send email about comment reply to Request.Email if it's set,
|
||||
// also sends email to site administrator if appropriate option is set.
|
||||
// Thread safe
|
||||
func (e *Email) Send(ctx context.Context, req Request) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("sending email messages about comment %q aborted due to canceled context", req.Comment.ID)
|
||||
default:
|
||||
}
|
||||
|
||||
var errs []error
|
||||
|
||||
for _, email := range req.Emails {
|
||||
err := e.buildAndSendMessage(ctx, req, email, false)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("problem sending user email notification to %q: %w", email, err))
|
||||
}
|
||||
}
|
||||
|
||||
for _, email := range e.AdminEmails {
|
||||
err := e.buildAndSendMessage(ctx, req, email, true)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("problem sending admin email notification to %q: %w", email, err))
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (e *Email) buildAndSendMessage(ctx context.Context, req Request, email string, forAdmin bool) error {
|
||||
log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
|
||||
msg, err := e.buildMessageFromRequest(req, email, forAdmin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return repeater.NewFixed(5, time.Millisecond*250).Do(
|
||||
ctx,
|
||||
func() error {
|
||||
return e.Email.Send(
|
||||
ctx,
|
||||
fmt.Sprintf("mailto:%s?from=%s&unsubscribeLink=%s&subject=%s",
|
||||
email,
|
||||
url.QueryEscape(e.From),
|
||||
url.QueryEscape(msg.unsubscribeLink),
|
||||
url.QueryEscape(msg.subject),
|
||||
),
|
||||
msg.body,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// SendVerification email verification VerificationRequest.Email if it's set.
|
||||
// Thread safe
|
||||
func (e *Email) SendVerification(ctx context.Context, req VerificationRequest) error {
|
||||
func (e *Email) Send(ctx context.Context, req Request) (err error) {
|
||||
if req.Email == "" {
|
||||
// this means we can't send this request via Email
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("sending message to %q aborted due to canceled context", req.User)
|
||||
return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
|
||||
default:
|
||||
}
|
||||
var msg string
|
||||
|
||||
log.Printf("[DEBUG] send verification via %s, user %s", e, req.User)
|
||||
msg, err := e.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
|
||||
if err != nil {
|
||||
return err
|
||||
if req.Verification.Token != "" {
|
||||
log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
|
||||
msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return repeater.NewFixed(5, time.Millisecond*250).Do(
|
||||
if req.Comment.ID != "" {
|
||||
if req.parent.User.ID == req.Comment.User.ID && !req.ForAdmin {
|
||||
// don't send anything if if user replied to their own comment
|
||||
return nil
|
||||
}
|
||||
log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
|
||||
msg, err = e.buildMessageFromRequest(req, req.ForAdmin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return repeater.NewDefault(5, time.Millisecond*250).Do(
|
||||
ctx,
|
||||
func() error {
|
||||
return e.Email.Send(
|
||||
ctx,
|
||||
fmt.Sprintf("mailto:%s?from=%s&subject=%s",
|
||||
req.Email,
|
||||
url.QueryEscape(e.From),
|
||||
url.QueryEscape(e.VerificationSubject),
|
||||
),
|
||||
msg,
|
||||
)
|
||||
return e.sendMessage(emailMessage{from: e.From, to: req.Email, message: msg})
|
||||
})
|
||||
}
|
||||
|
||||
// buildVerificationMessage generates verification email message based on given input
|
||||
func (e *Email) buildVerificationMessage(user, email, token, site string) (string, error) {
|
||||
subject := e.VerificationSubject
|
||||
msg := bytes.Buffer{}
|
||||
err := e.verifyTmpl.Execute(&msg, verifyTmplData{
|
||||
User: user,
|
||||
@@ -247,30 +290,24 @@ func (e *Email) buildVerificationMessage(user, email, token, site string) (strin
|
||||
SubscribeURL: e.SubscribeURL,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error executing template to build verification message: %w", err)
|
||||
return "", errors.Wrapf(err, "error executing template to build verification message")
|
||||
}
|
||||
return msg.String(), nil
|
||||
}
|
||||
|
||||
type commentMessage struct {
|
||||
subject string
|
||||
body string
|
||||
unsubscribeLink string
|
||||
return e.buildMessage(subject, msg.String(), email, "text/html", "")
|
||||
}
|
||||
|
||||
// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
|
||||
func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool) (commentMessage, error) {
|
||||
func (e *Email) buildMessageFromRequest(req Request, forAdmin bool) (string, error) {
|
||||
subject := "New reply to your comment"
|
||||
if forAdmin {
|
||||
subject = "New comment to your site"
|
||||
}
|
||||
if req.Comment.PostTitle != "" {
|
||||
subject += fmt.Sprintf(" for %q", req.Comment.PostTitle)
|
||||
subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
|
||||
}
|
||||
|
||||
token, err := e.TokenGenFn(req.parent.User.ID, email, req.Comment.Locator.SiteID)
|
||||
token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
|
||||
if err != nil {
|
||||
return commentMessage{}, fmt.Errorf("error creating token for unsubscribe link: %w", err)
|
||||
return "", errors.Wrapf(err, "error creating token for unsubscribe link")
|
||||
}
|
||||
unsubscribeLink := e.UnsubscribeURL + "?site=" + req.Comment.Locator.SiteID + "&tkn=" + token
|
||||
if forAdmin {
|
||||
@@ -282,11 +319,11 @@ func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool
|
||||
tmplData := msgTmplData{
|
||||
UserName: req.Comment.User.Name,
|
||||
UserPicture: req.Comment.User.Picture,
|
||||
CommentText: emailSafeHTML(req.Comment.Text),
|
||||
CommentText: req.Comment.Text,
|
||||
CommentLink: commentURLPrefix + req.Comment.ID,
|
||||
CommentDate: req.Comment.Timestamp,
|
||||
PostTitle: req.Comment.PostTitle,
|
||||
Email: email,
|
||||
Email: req.Email,
|
||||
UnsubscribeLink: unsubscribeLink,
|
||||
ForAdmin: forAdmin,
|
||||
}
|
||||
@@ -294,17 +331,146 @@ func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool
|
||||
if req.Comment.ParentID != "" {
|
||||
tmplData.ParentUserName = req.parent.User.Name
|
||||
tmplData.ParentUserPicture = req.parent.User.Picture
|
||||
tmplData.ParentCommentText = emailSafeHTML(req.parent.Text)
|
||||
tmplData.ParentCommentText = req.parent.Text
|
||||
tmplData.ParentCommentLink = commentURLPrefix + req.parent.ID
|
||||
tmplData.ParentCommentDate = req.parent.Timestamp
|
||||
}
|
||||
err = e.msgTmpl.Execute(&msg, tmplData)
|
||||
if err != nil {
|
||||
return commentMessage{}, fmt.Errorf("error executing template to build comment reply message: %w", err)
|
||||
return "", errors.Wrapf(err, "error executing template to build comment reply message")
|
||||
}
|
||||
return commentMessage{
|
||||
subject: subject,
|
||||
body: msg.String(),
|
||||
unsubscribeLink: unsubscribeLink,
|
||||
}, err
|
||||
return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
|
||||
}
|
||||
|
||||
// buildMessage generates email message to send using net/smtp.Data()
|
||||
func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink string) (message string, err error) {
|
||||
addHeader := func(msg, h, v string) string {
|
||||
msg += fmt.Sprintf("%s: %s\n", h, v)
|
||||
return msg
|
||||
}
|
||||
message = addHeader(message, "From", e.From)
|
||||
message = addHeader(message, "To", to)
|
||||
message = addHeader(message, "Subject", subject)
|
||||
message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
|
||||
|
||||
if contentType != "" {
|
||||
message = addHeader(message, "MIME-version", "1.0")
|
||||
message = addHeader(message, "Content-Type", contentType+`; charset="UTF-8"`)
|
||||
}
|
||||
|
||||
if unsubscribeLink != "" {
|
||||
// https://support.google.com/mail/answer/81126 -> "Include option to unsubscribe"
|
||||
message = addHeader(message, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
|
||||
message = addHeader(message, "List-Unsubscribe", "<"+unsubscribeLink+">")
|
||||
}
|
||||
|
||||
message = addHeader(message, "Date", time.Now().Format(time.RFC1123Z))
|
||||
|
||||
buff := &bytes.Buffer{}
|
||||
qp := quotedprintable.NewWriter(buff)
|
||||
if _, err := qp.Write([]byte(body)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// flush now, must NOT use defer, for small body, defer may cause buff.String() got empty body
|
||||
if err := qp.Close(); err != nil {
|
||||
return "", fmt.Errorf("quotedprintable Write failed: %w", err)
|
||||
}
|
||||
m := buff.String()
|
||||
message += "\n" + m
|
||||
return message, nil
|
||||
}
|
||||
|
||||
// sendMessage sends messages to server in a new connection, closing the connection after finishing.
|
||||
// Thread safe.
|
||||
func (e *Email) sendMessage(m emailMessage) error {
|
||||
if e.smtp == nil {
|
||||
return errors.New("sendMessage called without client set")
|
||||
}
|
||||
client, err := e.smtp.Create(e.SMTPParams)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to make smtp Create")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err = client.Quit(); err != nil {
|
||||
log.Printf("[WARN] failed to send quit command to %s:%d, %v", e.Host, e.Port, err)
|
||||
if err = client.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close smtp connection, %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err = client.Mail(m.from); err != nil {
|
||||
return errors.Wrapf(err, "bad from address %q", m.from)
|
||||
}
|
||||
if err = client.Rcpt(m.to); err != nil {
|
||||
return errors.Wrapf(err, "bad to address %q", m.to)
|
||||
}
|
||||
|
||||
writer, err := client.Data()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "can't make email writer")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err = writer.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close smtp body writer, %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
buf := bytes.NewBufferString(m.message)
|
||||
if _, err = buf.WriteTo(writer); err != nil {
|
||||
return errors.Wrapf(err, "failed to send email body to %q", m.to)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// String representation of Email object
|
||||
func (e *Email) String() string {
|
||||
return fmt.Sprintf("email: from %q with username '%s' at server %s:%d", e.From, e.Username, e.Host, e.Port)
|
||||
}
|
||||
|
||||
// Create establish SMTP connection with server using credentials in smtpClientWithCreator.SMTPParams
|
||||
// and returns pointer to it. Thread safe.
|
||||
func (s *emailClient) Create(params SMTPParams) (smtpClient, error) {
|
||||
authenticate := func(c *smtp.Client) error {
|
||||
if params.Username == "" || params.Password == "" {
|
||||
return nil
|
||||
}
|
||||
auth := smtp.PlainAuth("", params.Username, params.Password, params.Host)
|
||||
if err := c.Auth(auth); err != nil {
|
||||
return errors.Wrapf(err, "failed to auth to smtp %s:%d", params.Host, params.Port)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var c *smtp.Client
|
||||
srvAddress := fmt.Sprintf("%s:%d", params.Host, params.Port)
|
||||
if params.TLS {
|
||||
tlsConf := &tls.Config{
|
||||
InsecureSkipVerify: false,
|
||||
ServerName: params.Host,
|
||||
}
|
||||
conn, err := tls.Dial("tcp", srvAddress, tlsConf)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to dial smtp tls to %s", srvAddress)
|
||||
}
|
||||
if c, err = smtp.NewClient(conn, params.Host); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to make smtp client for %s", srvAddress)
|
||||
}
|
||||
return c, authenticate(c)
|
||||
}
|
||||
|
||||
conn, err := net.DialTimeout("tcp", srvAddress, params.TimeOut)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "timeout connecting to %s", srvAddress)
|
||||
}
|
||||
|
||||
c, err = smtp.NewClient(conn, params.Host)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to dial")
|
||||
}
|
||||
|
||||
return c, authenticate(c)
|
||||
}
|
||||
|
||||
+289
-206
@@ -1,98 +1,95 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"errors"
|
||||
"io"
|
||||
"net/smtp"
|
||||
"sync"
|
||||
"testing"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
func TestEmailNew(t *testing.T) {
|
||||
emailParams := EmailParams{
|
||||
From: "test@from",
|
||||
VerificationTemplatePath: "testdata/verification.html.tmpl",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
}
|
||||
smtpParams := ntf.SMTPParams{
|
||||
Host: "test@host",
|
||||
Port: 1000,
|
||||
TLS: true,
|
||||
StartTLS: true,
|
||||
Username: "test@username",
|
||||
Password: "test@password",
|
||||
}
|
||||
|
||||
email, err := NewEmail(emailParams, smtpParams)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, email, "email returned")
|
||||
|
||||
assert.NotNil(t, email.msgTmpl, "e.template is set")
|
||||
assert.Equal(t, emailParams.From, email.From, "emailParams.From unchanged after creation")
|
||||
if smtpParams.TimeOut == 0 {
|
||||
assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
|
||||
} else {
|
||||
assert.Equal(t, smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
|
||||
}
|
||||
assert.Equal(t, smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
|
||||
assert.Equal(t, smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
|
||||
assert.Equal(t, smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
|
||||
assert.Equal(t, smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
|
||||
assert.Equal(t, smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
|
||||
assert.Equal(t, smtpParams.StartTLS, email.StartTLS, "emailParams.TLS unchanged after creation")
|
||||
assert.Equal(t, "email: with username 'test@username' at server test@host:1000 with TLS with StartTLS", email.String())
|
||||
}
|
||||
|
||||
func Test_initTemplatesErr(t *testing.T) {
|
||||
testSet := []struct {
|
||||
var testSet = []struct {
|
||||
name string
|
||||
err bool
|
||||
errText string
|
||||
emailParams EmailParams
|
||||
smtpParams SMTPParams
|
||||
}{
|
||||
{
|
||||
name: "with wrong path to verification template",
|
||||
errText: "notfound.tmpl: file does not exist",
|
||||
{name: "empty"},
|
||||
{name: "with template parse error",
|
||||
err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
|
||||
emailParams: EmailParams{
|
||||
VerificationTemplatePath: "notfound.tmpl",
|
||||
MsgTemplate: "{{",
|
||||
}},
|
||||
{name: "with verification template parse error",
|
||||
err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
|
||||
emailParams: EmailParams{
|
||||
From: "test@from",
|
||||
VerificationTemplate: "{{",
|
||||
},
|
||||
smtpParams: SMTPParams{
|
||||
Host: "test@host",
|
||||
Port: 1000,
|
||||
TLS: true,
|
||||
Username: "test@username",
|
||||
Password: "test@password",
|
||||
TimeOut: time.Second,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with wrong path to message template",
|
||||
errText: "notfound.tmpl: file does not exist",
|
||||
{name: "normal creation",
|
||||
err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
|
||||
emailParams: EmailParams{
|
||||
MsgTemplatePath: "notfound.tmpl",
|
||||
From: "test@from",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with error on read verification template",
|
||||
errText: "can't parse verification template: template: verifyTmpl",
|
||||
emailParams: EmailParams{
|
||||
VerificationTemplatePath: "testdata/bad.html.tmpl",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with error on read message template",
|
||||
errText: "can't parse message template: template: msgTmpl",
|
||||
emailParams: EmailParams{
|
||||
VerificationTemplatePath: "testdata/verification.html.tmpl",
|
||||
MsgTemplatePath: "testdata/bad.html.tmpl",
|
||||
smtpParams: SMTPParams{
|
||||
Host: "test@host",
|
||||
Port: 1000,
|
||||
TLS: true,
|
||||
Username: "test@username",
|
||||
Password: "test@password",
|
||||
TimeOut: time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, d := range testSet {
|
||||
d := d
|
||||
t.Run(d.name, func(t *testing.T) {
|
||||
e, err := NewEmail(d.emailParams, ntf.SMTPParams{})
|
||||
require.Error(t, err)
|
||||
require.Nil(t, e)
|
||||
assert.Contains(t, err.Error(), d.errText)
|
||||
email, err := NewEmail(d.emailParams, d.smtpParams)
|
||||
|
||||
if d.err && d.errText == "" {
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, email)
|
||||
} else if d.err && d.errText != "" {
|
||||
assert.EqualError(t, err, d.errText)
|
||||
assert.Nil(t, email)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, email, "email returned")
|
||||
|
||||
assert.NotNil(t, email.msgTmpl, "e.template is set")
|
||||
assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
|
||||
assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
|
||||
assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
|
||||
if d.smtpParams.TimeOut == 0 {
|
||||
assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
|
||||
} else {
|
||||
assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
|
||||
}
|
||||
assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
|
||||
assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
|
||||
assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
|
||||
assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
|
||||
assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -104,192 +101,278 @@ func TestEmailSendErrors(t *testing.T) {
|
||||
|
||||
e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualError(t, e.SendVerification(context.Background(), VerificationRequest{Email: "bad@example.org", Token: "some"}),
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
|
||||
"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
|
||||
e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
|
||||
assert.NoError(t, err)
|
||||
|
||||
e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
|
||||
"problem sending user email notification to \"bad@example.org\": "+
|
||||
"error executing template to build comment reply message: "+
|
||||
"template: test:1:2: executing \"test\" at <.Test>: "+
|
||||
"can't evaluate field Test in type notify.msgTmplData")
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
|
||||
"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
|
||||
e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
|
||||
"sending email messages about comment \"999\" aborted due to canceled context")
|
||||
assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
|
||||
"sending message to \"bad@example.org\" aborted due to canceled context")
|
||||
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad@example.org"}}),
|
||||
"problem sending user email notification to \"bad@example.org\":"+
|
||||
" error creating token for unsubscribe link: token generation error")
|
||||
|
||||
// errors for all failed recipients are reported, not just the last one
|
||||
assert.EqualError(t, e.Send(context.Background(),
|
||||
Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad1@example.org", "bad2@example.org"}}),
|
||||
"problem sending user email notification to \"bad1@example.org\": error creating token for unsubscribe link: token generation error\n"+
|
||||
"problem sending user email notification to \"bad2@example.org\": error creating token for unsubscribe link: token generation error")
|
||||
e.smtp = &fakeTestSMTP{}
|
||||
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
|
||||
"error creating token for unsubscribe link: token generation error")
|
||||
e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestEmailSend_ExitConditions(t *testing.T) {
|
||||
email, err := NewEmail(EmailParams{
|
||||
VerificationTemplatePath: "testdata/verification.html.tmpl",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
}, ntf.SMTPParams{})
|
||||
email, err := NewEmail(EmailParams{}, SMTPParams{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, email, "expecting email returned")
|
||||
// prevent triggering e.autoFlush creation
|
||||
emptyRequest := Request{Comment: store.Comment{ID: "999"}}
|
||||
assert.NoError(t, email.Send(context.Background(), emptyRequest),
|
||||
"Message without Emails and AdminEmails is not sent and returns nil")
|
||||
"Message without parent comment User.Email is not sent and returns nil")
|
||||
requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
|
||||
assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
|
||||
"Message with parent comment User equals comment User is not sent and returns nil")
|
||||
}
|
||||
|
||||
func TestEmailSendClientError(t *testing.T) {
|
||||
var testSet = []struct {
|
||||
name string
|
||||
smtp *fakeTestSMTP
|
||||
err string
|
||||
}{
|
||||
{name: "failed to verify receiver", smtp: &fakeTestSMTP{fail: map[string]bool{"mail": true}},
|
||||
err: "bad from address \"\": failed to verify sender"},
|
||||
{name: "failed to verify sender", smtp: &fakeTestSMTP{fail: map[string]bool{"rcpt": true}},
|
||||
err: "bad to address \"\": failed to verify receiver"},
|
||||
{name: "failed to close connection", smtp: &fakeTestSMTP{fail: map[string]bool{"quit": true, "close": true}}},
|
||||
{name: "failed to make email writer", smtp: &fakeTestSMTP{fail: map[string]bool{"data": true}},
|
||||
err: "can't make email writer: failed to send"},
|
||||
}
|
||||
for _, d := range testSet {
|
||||
d := d
|
||||
t.Run(d.name, func(t *testing.T) {
|
||||
e := Email{smtp: d.smtp}
|
||||
if d.err != "" {
|
||||
assert.EqualError(t, e.sendMessage(emailMessage{}), d.err,
|
||||
"expected error for e.sendMessage")
|
||||
} else {
|
||||
assert.NoError(t, e.sendMessage(emailMessage{}),
|
||||
"expected no error for e.sendMessage")
|
||||
}
|
||||
})
|
||||
}
|
||||
e := Email{}
|
||||
e.smtp = nil
|
||||
assert.Error(t, e.sendMessage(emailMessage{}),
|
||||
"nil e.smtp should return error")
|
||||
e.smtp = &fakeTestSMTP{}
|
||||
assert.NoError(t, e.sendMessage(emailMessage{}), "",
|
||||
"no error expected for e.sendMessage in normal flow")
|
||||
e.smtp = &fakeTestSMTP{fail: map[string]bool{"quit": true}}
|
||||
assert.NoError(t, e.sendMessage(emailMessage{}), "",
|
||||
"no error expected for e.sendMessage with failed smtpClient.Quit but successful smtpClient.Close")
|
||||
e.smtp = &fakeTestSMTP{fail: map[string]bool{"create": true}}
|
||||
assert.EqualError(t, e.sendMessage(emailMessage{}), "failed to make smtp Create: failed to create client",
|
||||
"e.send called without smtpClient set returns error")
|
||||
}
|
||||
|
||||
func TestEmail_Send(t *testing.T) {
|
||||
email, err := NewEmail(EmailParams{
|
||||
From: "from@example.org",
|
||||
VerificationTemplatePath: "testdata/verification.html.tmpl",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
}, ntf.SMTPParams{})
|
||||
email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, email)
|
||||
fakeSMTP := fakeTestSMTP{}
|
||||
email.smtp = &fakeSMTP
|
||||
email.TokenGenFn = TokenGenFn
|
||||
email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
|
||||
req := Request{
|
||||
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
|
||||
parent: store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
|
||||
Emails: []string{"test@example.org"},
|
||||
Email: "test@example.org",
|
||||
}
|
||||
assert.Contains(t, email.Send(context.Background(), req).Error(), "problem sending user email notification to \"test@example.org\"")
|
||||
assert.NoError(t, email.Send(context.TODO(), req))
|
||||
assert.Equal(t, "from@example.org", fakeSMTP.readMail())
|
||||
assert.Equal(t, 1, fakeSMTP.readQuitCount())
|
||||
assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
|
||||
// test buildMessageFromRequest separately for message text
|
||||
msg, err := email.buildMessageFromRequest(req, req.Emails[0], false)
|
||||
res, err := email.buildMessageFromRequest(req, req.ForAdmin)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, `
|
||||
New reply from test_user on your comment to «test_title»
|
||||
assert.Contains(t, res, `From: from@example.org
|
||||
To: test@example.org
|
||||
Subject: New reply to your comment for "test_title"
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
MIME-version: 1.0
|
||||
Content-Type: text/html; charset="UTF-8"
|
||||
List-Unsubscribe-Post: List-Unsubscribe=One-Click
|
||||
List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
|
||||
Date: `)
|
||||
|
||||
User: test_user
|
||||
01.01.0001 at 00:00
|
||||
Comment:
|
||||
test@example.org for parent_user
|
||||
Unsubscribe link: https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token
|
||||
`, msg.body)
|
||||
assert.Equal(t, "https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token", msg.unsubscribeLink)
|
||||
assert.Equal(t, `New reply to your comment for "test_title"`, msg.subject)
|
||||
|
||||
// send email to both user and admin, without parent set
|
||||
email.AdminEmails = []string{"admin@example.org"}
|
||||
// send email to admin without parent set
|
||||
req = Request{
|
||||
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
|
||||
Emails: []string{"test@example.org"},
|
||||
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
|
||||
Email: "admin@example.org",
|
||||
ForAdmin: true,
|
||||
}
|
||||
assert.Error(t, email.Send(context.Background(), req))
|
||||
msg, err = email.buildMessageFromRequest(req, email.AdminEmails[0], true)
|
||||
assert.NoError(t, email.Send(context.TODO(), req))
|
||||
res, err = email.buildMessageFromRequest(req, req.ForAdmin)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, `
|
||||
New comment from test_user on your site to «test_title»
|
||||
|
||||
User: test_user
|
||||
01.01.0001 at 00:00
|
||||
Comment:
|
||||
admin@example.org
|
||||
`, msg.body)
|
||||
assert.Equal(t, `New comment to your site for "test_title"`, msg.subject)
|
||||
assert.Empty(t, msg.unsubscribeLink)
|
||||
}
|
||||
|
||||
func TestEmail_CommentTextSanitizedForEmail(t *testing.T) {
|
||||
// comment HTML reaching the email path is sanitized by the store-level UGC policy,
|
||||
// which permits <a> and <img>. The email must drop both so a comment can't inject
|
||||
// phishing links or remote tracking pixels into a notification (GHSA-74pc-3r2m-ppx3).
|
||||
email, err := NewEmail(EmailParams{
|
||||
From: "from@example.org",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
}, ntf.SMTPParams{})
|
||||
require.NoError(t, err)
|
||||
email.TokenGenFn = TokenGenFn
|
||||
|
||||
malicious := `hello <a href="https://phishing.example/verify">click to verify</a>` +
|
||||
` <img src="https://attacker.example/track.png" width="1" height="1"> <b>kept</b>`
|
||||
req := Request{
|
||||
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title", Text: malicious},
|
||||
Emails: []string{"test@example.org"},
|
||||
}
|
||||
msg, err := email.buildMessageFromRequest(req, req.Emails[0], false)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotContains(t, msg.body, "phishing.example", "phishing link must be stripped")
|
||||
assert.NotContains(t, msg.body, "attacker.example", "tracking pixel must be stripped")
|
||||
assert.NotContains(t, msg.body, "<img", "no image tags in email body")
|
||||
assert.NotContains(t, msg.body, "<a ", "no anchor tags in email body")
|
||||
assert.Contains(t, msg.body, "click to verify", "anchor text is preserved, only the link is dropped")
|
||||
assert.Contains(t, msg.body, "<b>kept</b>", "basic formatting is preserved")
|
||||
}
|
||||
|
||||
// emailSafeHTML drops links/images while keeping inline/block formatting and escaping nothing extra.
|
||||
func TestEmailSafeHTML(t *testing.T) {
|
||||
tbl := []struct{ name, in, want string }{
|
||||
{"strips anchor keeps text", `<a href="http://evil">x</a>`, "x"},
|
||||
{"strips image entirely", `a<img src="http://evil/t.png">b`, "ab"},
|
||||
{"keeps bold/italic/code", `<b>b</b><i>i</i><code>c</code>`, `<b>b</b><i>i</i><code>c</code>`},
|
||||
{"keeps blockquote and lists", `<blockquote>q</blockquote><ul><li>x</li></ul>`, `<blockquote>q</blockquote><ul><li>x</li></ul>`},
|
||||
{"drops onclick handlers", `<span onclick="alert(1)">s</span>`, `<span>s</span>`},
|
||||
}
|
||||
for _, tt := range tbl {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, string(emailSafeHTML(tt.in)))
|
||||
})
|
||||
}
|
||||
assert.Contains(t, res, `From: from@example.org
|
||||
To: admin@example.org
|
||||
Subject: New comment to your site for "test_title"
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
MIME-version: 1.0
|
||||
Content-Type: text/html; charset="UTF-8"
|
||||
Date: `)
|
||||
}
|
||||
|
||||
func TestEmail_SendVerification(t *testing.T) {
|
||||
email, err := NewEmail(EmailParams{
|
||||
From: "from@example.org",
|
||||
VerificationTemplatePath: "testdata/verification.html.tmpl",
|
||||
MsgTemplatePath: "testdata/msg.html.tmpl",
|
||||
}, ntf.SMTPParams{})
|
||||
email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, email)
|
||||
fakeSMTP := fakeTestSMTP{}
|
||||
email.smtp = &fakeSMTP
|
||||
email.TokenGenFn = TokenGenFn
|
||||
// proper VerificationRequest without email
|
||||
req := VerificationRequest{
|
||||
SiteID: "remark",
|
||||
User: "test_username",
|
||||
Token: "secret_",
|
||||
req := Request{
|
||||
Email: "test@example.org",
|
||||
Verification: VerificationMetadata{
|
||||
SiteID: "remark",
|
||||
User: "test_username",
|
||||
Token: "secret_",
|
||||
},
|
||||
}
|
||||
assert.NoError(t, email.SendVerification(context.Background(), req))
|
||||
|
||||
// proper VerificationRequest with email
|
||||
req.Email = "test@example.org"
|
||||
assert.Error(t, email.SendVerification(context.Background(), req), "failed to make smtp client")
|
||||
|
||||
// VerificationRequest with canceled context
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
assert.EqualError(t, email.SendVerification(ctx, req), "sending message to \"test_username\" aborted due to canceled context")
|
||||
|
||||
// test buildVerificationMessage separately for message text
|
||||
res, err := email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
|
||||
assert.NoError(t, email.Send(context.TODO(), req))
|
||||
assert.Equal(t, "from@example.org", fakeSMTP.readMail())
|
||||
assert.Equal(t, 1, fakeSMTP.readQuitCount())
|
||||
assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
|
||||
// test buildMessageFromRequest separately for message text
|
||||
res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res, `Confirmation for test_username on site remark
|
||||
Token:secret_
|
||||
Sent to test@example.org
|
||||
|
||||
`)
|
||||
assert.Contains(t, res, `From: from@example.org
|
||||
To: test@example.org
|
||||
Subject: Email verification
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
MIME-version: 1.0
|
||||
Content-Type: text/html; charset="UTF-8"
|
||||
Date: `)
|
||||
assert.Contains(t, res, `secret_`)
|
||||
assert.NotContains(t, res, `https://example.org/`)
|
||||
email.SubscribeURL = "https://example.org/subscribe.html?token="
|
||||
res, err = email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
|
||||
res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res, `Confirmation for test_username on site remark
|
||||
Subscribe url: https://example.org/subscribe.html?token=secret_
|
||||
Token:secret_
|
||||
Sent to test@example.org
|
||||
assert.Contains(t, res, `From: from@example.org
|
||||
To: test@example.org
|
||||
Subject: Email verification
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
MIME-version: 1.0
|
||||
Content-Type: text/html; charset="UTF-8"
|
||||
Date: `)
|
||||
assert.Contains(t, res, `https://example.org/subscribe.html?token=3Dsecret_`)
|
||||
}
|
||||
|
||||
`)
|
||||
func Test_emailClient_Create(t *testing.T) {
|
||||
creator := emailClient{}
|
||||
client, err := creator.Create(SMTPParams{})
|
||||
assert.Error(t, err, "absence of address to connect results in error")
|
||||
assert.Nil(t, client, "no client returned in case of error")
|
||||
}
|
||||
|
||||
type fakeTestSMTP struct {
|
||||
fail map[string]bool
|
||||
|
||||
buff bytes.Buffer
|
||||
mail, rcpt string
|
||||
auth bool
|
||||
close bool
|
||||
quitCount int
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Create(SMTPParams) (smtpClient, error) {
|
||||
if f.fail["create"] {
|
||||
return nil, errors.New("failed to create client")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Auth(smtp.Auth) error { f.auth = true; return nil }
|
||||
|
||||
func (f *fakeTestSMTP) Mail(m string) error {
|
||||
f.lock.Lock()
|
||||
f.mail = m
|
||||
f.lock.Unlock()
|
||||
if f.fail["mail"] {
|
||||
return errors.New("failed to verify sender")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Rcpt(r string) error {
|
||||
f.lock.Lock()
|
||||
f.rcpt = r
|
||||
f.lock.Unlock()
|
||||
if f.fail["rcpt"] {
|
||||
return errors.New("failed to verify receiver")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Quit() error {
|
||||
f.lock.Lock()
|
||||
f.quitCount++
|
||||
f.lock.Unlock()
|
||||
if f.fail["quit"] {
|
||||
return errors.New("failed to quit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Close() error {
|
||||
f.close = true
|
||||
if f.fail["close"] {
|
||||
return errors.New("failed to close")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) Data() (io.WriteCloser, error) {
|
||||
if f.fail["data"] {
|
||||
return nil, errors.New("failed to send")
|
||||
}
|
||||
return nopCloser{&f.buff}, nil
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) readRcpt() string {
|
||||
f.lock.RLock()
|
||||
defer f.lock.RUnlock()
|
||||
return f.rcpt
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) readMail() string {
|
||||
f.lock.RLock()
|
||||
defer f.lock.RUnlock()
|
||||
return f.mail
|
||||
}
|
||||
|
||||
func (f *fakeTestSMTP) readQuitCount() int {
|
||||
f.lock.RLock()
|
||||
defer f.lock.RUnlock()
|
||||
return f.quitCount
|
||||
}
|
||||
|
||||
func TokenGenFn(user, _, _ string) (string, error) {
|
||||
if user == "error" {
|
||||
return "", fmt.Errorf("token generation error")
|
||||
return "", errors.New("token generation error")
|
||||
}
|
||||
return "token", nil
|
||||
}
|
||||
|
||||
type nopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (nopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
+44
-129
@@ -9,17 +9,16 @@ import (
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
// Service delivers notifications to multiple destinations
|
||||
type Service struct {
|
||||
dataService Store
|
||||
destinations []Destination
|
||||
queue chan Request
|
||||
verificationQueue chan VerificationRequest
|
||||
dataService Store
|
||||
destinations []Destination
|
||||
queue chan Request
|
||||
|
||||
closed atomic.Uint32 // non-zero means closed. uses uint instead of bool for atomic
|
||||
closed uint32 // non-zero means closed. uses uint instead of bool for atomic
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@@ -27,33 +26,29 @@ type Service struct {
|
||||
// Destination defines interface for a given destination service, like telegram, email and so on
|
||||
type Destination interface {
|
||||
fmt.Stringer
|
||||
Send(context.Context, Request) error
|
||||
SendVerification(context.Context, VerificationRequest) error
|
||||
Send(ctx context.Context, req Request) error
|
||||
}
|
||||
|
||||
// Store defines the minimal interface accessing stored comments used by notifier
|
||||
type Store interface {
|
||||
Get(locator store.Locator, id string, user store.User) (store.Comment, error)
|
||||
GetUserEmail(siteID, userID string) (string, error)
|
||||
GetUserTelegram(siteID, userID string) (string, error)
|
||||
GetUserEmail(siteID string, userID string) (string, error)
|
||||
}
|
||||
|
||||
// used for email and telegram retrieval from user details
|
||||
type getUserDetail func(string, string) (string, error)
|
||||
|
||||
// Request notification for a Comment
|
||||
// Request notification either about comment or about particular user verification
|
||||
type Request struct {
|
||||
Comment store.Comment
|
||||
parent store.Comment
|
||||
Emails []string
|
||||
Telegrams []string
|
||||
Comment store.Comment // if set sent notifications about new comment
|
||||
parent store.Comment // fetched only in case Comment is set
|
||||
Email string // if set (also) send email
|
||||
ForAdmin bool // if set, message supposed to be sent to administrator
|
||||
|
||||
Verification VerificationMetadata // if set sent verification notification
|
||||
}
|
||||
|
||||
// VerificationRequest notification for user
|
||||
type VerificationRequest struct {
|
||||
// VerificationMetadata required to send notify method verification message
|
||||
type VerificationMetadata struct {
|
||||
SiteID string
|
||||
User string
|
||||
Email string // if set, send email only
|
||||
Token string
|
||||
}
|
||||
|
||||
@@ -67,12 +62,11 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
res := Service{
|
||||
dataService: dataService,
|
||||
queue: make(chan Request, size),
|
||||
verificationQueue: make(chan VerificationRequest, size),
|
||||
destinations: destinations,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
dataService: dataService,
|
||||
queue: make(chan Request, size),
|
||||
destinations: destinations,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
if len(destinations) > 0 {
|
||||
go res.do()
|
||||
@@ -83,14 +77,21 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
|
||||
|
||||
// Submit Request to internal channel if not busy, drop if can't send
|
||||
func (s *Service) Submit(req Request) {
|
||||
if len(s.destinations) == 0 || s.closed.Load() != 0 {
|
||||
if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
|
||||
return
|
||||
}
|
||||
// parent comment is fetched only if comment is present in the Request
|
||||
if s.dataService != nil && req.Comment.ParentID != "" {
|
||||
if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
|
||||
req.parent = p
|
||||
req.Emails = s.getNotificationTargets(req, p, s.dataService.GetUserEmail)
|
||||
req.Telegrams = s.getNotificationTargets(req, p, s.dataService.GetUserTelegram)
|
||||
// user notification, should fetch email for it.
|
||||
// administrator notification comes with pre-set email
|
||||
if req.Email == "" {
|
||||
req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
@@ -100,119 +101,33 @@ func (s *Service) Submit(req Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// getNotificationTargets returns list of notification targets (like email or telegram username) for users
|
||||
// interested in notifications for provided comment.
|
||||
// Targets are not added to the returned list in case the original message
|
||||
// is from the same user as the notification receiver.
|
||||
// Results are deduplicated.
|
||||
func (s *Service) getNotificationTargets(
|
||||
req Request,
|
||||
notifyComment store.Comment,
|
||||
getUserDetail getUserDetail,
|
||||
) (result []string) {
|
||||
// add current user email only if the user is not the one who wrote the original comment
|
||||
if notifyComment.User.ID != req.Comment.User.ID {
|
||||
detail, err := getUserDetail(req.Comment.Locator.SiteID, notifyComment.User.ID)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] can't read notification detail for %s, %v", notifyComment.User.ID, err)
|
||||
}
|
||||
if detail != "" {
|
||||
result = append(result, detail)
|
||||
}
|
||||
}
|
||||
if notifyComment.ParentID != "" {
|
||||
if p, err := s.dataService.Get(req.Comment.Locator, notifyComment.ParentID, store.User{}); err == nil {
|
||||
result = append(result, s.getNotificationTargets(req, p, getUserDetail)...)
|
||||
}
|
||||
}
|
||||
return deduplicateStrings(result)
|
||||
}
|
||||
|
||||
// SubmitVerification to internal channel if not busy, drop if can't send
|
||||
func (s *Service) SubmitVerification(req VerificationRequest) {
|
||||
if len(s.destinations) == 0 || s.closed.Load() != 0 {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.verificationQueue <- req:
|
||||
default:
|
||||
log.Printf("[WARN] can't send verification to queue, %s for %s", req.User, req.Email)
|
||||
}
|
||||
}
|
||||
|
||||
// Close queue channel and wait for completion
|
||||
func (s *Service) Close() {
|
||||
if s.queue != nil {
|
||||
// don't panic in case service is already closed
|
||||
select {
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
log.Print("[DEBUG] close notifier")
|
||||
close(s.queue)
|
||||
close(s.verificationQueue)
|
||||
s.cancel()
|
||||
<-s.ctx.Done()
|
||||
}
|
||||
s.closed.Store(1)
|
||||
atomic.StoreUint32(&s.closed, 1)
|
||||
}
|
||||
|
||||
func (s *Service) do() {
|
||||
defer log.Print("[WARN] terminated notifier")
|
||||
var wg sync.WaitGroup
|
||||
for {
|
||||
select {
|
||||
case c, ok := <-s.queue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
wg.Add(len(s.destinations))
|
||||
for _, dest := range s.destinations {
|
||||
go func(d Destination) {
|
||||
if err := d.Send(s.ctx, c); err != nil {
|
||||
log.Printf("[WARN] failed to send to %s, %s", d, err)
|
||||
}
|
||||
wg.Done()
|
||||
}(dest)
|
||||
}
|
||||
wg.Wait()
|
||||
case v, ok := <-s.verificationQueue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
wg.Add(len(s.destinations))
|
||||
for _, dest := range s.destinations {
|
||||
go func(d Destination) {
|
||||
if err := d.SendVerification(s.ctx, v); err != nil {
|
||||
log.Printf("[WARN] failed to send to %s, %s", d, err)
|
||||
}
|
||||
wg.Done()
|
||||
}(dest)
|
||||
}
|
||||
wg.Wait()
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
for c := range s.queue {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(s.destinations))
|
||||
for _, dest := range s.destinations {
|
||||
go func(d Destination) {
|
||||
if err := d.Send(s.ctx, c); err != nil {
|
||||
log.Printf("[WARN] failed to send to %s, %s", d, err)
|
||||
}
|
||||
wg.Done()
|
||||
}(dest)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
log.Print("[WARN] terminated notifier")
|
||||
}
|
||||
|
||||
// NopService is do-nothing notifier, without destinations
|
||||
var NopService = &Service{}
|
||||
|
||||
// deduplicateStrings returns provided slice of strings will all duplicates removed.
|
||||
// Resulting slice is not sorted.
|
||||
func deduplicateStrings(source []string) []string {
|
||||
set := make(map[string]struct{})
|
||||
|
||||
for _, k := range source {
|
||||
set[k] = struct{}{}
|
||||
}
|
||||
|
||||
result := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
result = append(result, k)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -4,51 +4,31 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
)
|
||||
|
||||
// MockDest is a destination mock
|
||||
type MockDest struct {
|
||||
data []Request
|
||||
verificationData []VerificationRequest
|
||||
id int
|
||||
closed bool
|
||||
lock sync.Mutex
|
||||
block chan struct{} // if non-nil, Send/SendVerification wait on it before recording, letting tests pin the consumer
|
||||
data []Request
|
||||
id int
|
||||
closed bool
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
// Send mock
|
||||
func (m *MockDest) Send(ctx context.Context, r Request) error {
|
||||
if m.block != nil {
|
||||
<-m.block
|
||||
}
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
select {
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
m.data = append(m.data, r)
|
||||
log.Printf("sent %s -> %d", r.Comment.ID, m.id)
|
||||
case <-ctx.Done():
|
||||
log.Printf("ctx closed %d", m.id)
|
||||
m.closed = true
|
||||
return nil
|
||||
}
|
||||
m.data = append(m.data, r)
|
||||
log.Printf("sent %s -> %d", r.Comment.ID, m.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SendVerification mock
|
||||
func (m *MockDest) SendVerification(ctx context.Context, v VerificationRequest) error {
|
||||
if m.block != nil {
|
||||
<-m.block
|
||||
}
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("verification ctx closed %d", m.id)
|
||||
m.closed = true
|
||||
return nil
|
||||
}
|
||||
m.verificationData = append(m.verificationData, v)
|
||||
log.Printf("sent verification %s -> %d", v.User, m.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -60,25 +40,4 @@ func (m *MockDest) Get() []Request {
|
||||
copy(res, m.data)
|
||||
return res
|
||||
}
|
||||
|
||||
// GetVerify mock
|
||||
func (m *MockDest) GetVerify() []VerificationRequest {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
res := make([]VerificationRequest, len(m.verificationData))
|
||||
copy(res, m.verificationData)
|
||||
return res
|
||||
}
|
||||
|
||||
// IsClosed returns closed status safely
|
||||
func (m *MockDest) IsClosed() bool {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
return m.closed
|
||||
}
|
||||
|
||||
func (m *MockDest) String() string {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed)
|
||||
}
|
||||
func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
func TestService_NoDestinations(t *testing.T) {
|
||||
@@ -19,312 +22,107 @@ func TestService_NoDestinations(t *testing.T) {
|
||||
s.Submit(Request{Comment: store.Comment{ID: "123"}})
|
||||
s.Submit(Request{Comment: store.Comment{ID: "123"}})
|
||||
s.Close()
|
||||
// second call should not result in panic
|
||||
s.Close()
|
||||
}
|
||||
|
||||
func TestService_WithDestinations(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}})
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}})
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}})
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}})
|
||||
time.Sleep(time.Millisecond * 110)
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}})
|
||||
time.Sleep(time.Millisecond * 110)
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}})
|
||||
time.Sleep(time.Millisecond * 110)
|
||||
s.Close()
|
||||
|
||||
require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
|
||||
require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
|
||||
require.Equal(t, 3, len(d1.Get()), "got all comments to d1")
|
||||
require.Equal(t, 3, len(d2.Get()), "got all comments to d2")
|
||||
|
||||
assert.Equal(t, "100", d1.Get()[0].Comment.ID)
|
||||
assert.Equal(t, "101", d1.Get()[1].Comment.ID)
|
||||
assert.Equal(t, "102", d1.Get()[2].Comment.ID)
|
||||
})
|
||||
assert.Equal(t, "100", d1.Get()[0].Comment.ID)
|
||||
assert.Equal(t, "101", d1.Get()[1].Comment.ID)
|
||||
assert.Equal(t, "102", d1.Get()[2].Comment.ID)
|
||||
}
|
||||
|
||||
func TestService_WithDrops(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// gated destinations pin the consumer on the first item so the size-1 queue
|
||||
// fills deterministically and the overflow is dropped regardless of scheduling
|
||||
gate := make(chan struct{})
|
||||
d1, d2 := &MockDest{id: 1, block: gate}, &MockDest{id: 2, block: gate}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}}) // consumed, consumer blocks in Send on the gate
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}}) // fills the size-1 queue
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}}) // queue full, dropped
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "100"}})
|
||||
s.Submit(Request{Comment: store.Comment{ID: "101"}})
|
||||
time.Sleep(time.Millisecond * 11)
|
||||
s.Submit(Request{Comment: store.Comment{ID: "102"}})
|
||||
time.Sleep(time.Millisecond * 11)
|
||||
s.Close()
|
||||
|
||||
close(gate) // release the consumer: it finishes 100 then processes 101
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
|
||||
|
||||
require.Len(t, d1.Get(), 2, "one comment of three dropped from d1, got: %v", d1.Get())
|
||||
require.Len(t, d2.Get(), 2, "one comment of three dropped from d2, got: %v", d2.Get())
|
||||
assert.Equal(t, "100", d1.Get()[0].Comment.ID)
|
||||
assert.Equal(t, "101", d1.Get()[1].Comment.ID)
|
||||
})
|
||||
}
|
||||
|
||||
func TestService_SubmitVerificationWithDrops(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// gated destinations pin the consumer on the first item so the size-1 queue
|
||||
// fills deterministically and the overflow is dropped regardless of scheduling
|
||||
gate := make(chan struct{})
|
||||
d1, d2 := &MockDest{id: 1, block: gate}, &MockDest{id: 2, block: gate}
|
||||
s := NewService(nil, 1, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
s.SubmitVerification(VerificationRequest{
|
||||
SiteID: "remark",
|
||||
User: "testUser",
|
||||
Email: "test@example.org",
|
||||
Token: "testToken",
|
||||
}) // consumed, consumer blocks in SendVerification on the gate
|
||||
synctest.Wait()
|
||||
s.SubmitVerification(VerificationRequest{User: "second"}) // fills the size-1 queue
|
||||
s.SubmitVerification(VerificationRequest{User: "dropped"}) // queue full, dropped
|
||||
synctest.Wait()
|
||||
|
||||
close(gate) // release the consumer: it finishes testUser then processes second
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
|
||||
s.SubmitVerification(VerificationRequest{}) // safe to send after close
|
||||
|
||||
require.Len(t, d2.GetVerify(), 2, "one request of three dropped from d2, got: %v", d2.GetVerify())
|
||||
|
||||
verifyDest := d1.GetVerify()
|
||||
require.Len(t, verifyDest, 2, "one request of three dropped from d1, got: %v", verifyDest)
|
||||
assert.Equal(t, "remark", verifyDest[0].SiteID)
|
||||
assert.Equal(t, "testUser", verifyDest[0].User)
|
||||
assert.Equal(t, "test@example.org", verifyDest[0].Email)
|
||||
assert.Equal(t, "testToken", verifyDest[0].Token)
|
||||
assert.Equal(t, "second", verifyDest[1].User)
|
||||
})
|
||||
assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
|
||||
assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
|
||||
}
|
||||
|
||||
func TestService_Many(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
s := NewService(nil, 5, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
|
||||
s := NewService(nil, 5, d1, d2)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
for i := range 10 {
|
||||
s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
|
||||
s.SubmitVerification(VerificationRequest{User: fmt.Sprintf("%d", 100+i)})
|
||||
}
|
||||
s.Close()
|
||||
for i := 0; i < 10; i++ {
|
||||
s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
|
||||
time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
|
||||
}
|
||||
s.Close()
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
|
||||
assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
|
||||
assert.NotEqual(t, 10, len(d1.GetVerify()), "some verifications dropped from d1")
|
||||
assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
|
||||
assert.NotEqual(t, 10, len(d2.GetVerify()), "some verifications dropped from d2")
|
||||
})
|
||||
assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
|
||||
assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
|
||||
|
||||
assert.True(t, d1.closed)
|
||||
assert.True(t, d2.closed)
|
||||
}
|
||||
|
||||
func TestService_WithParent(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dest := &MockDest{id: 1}
|
||||
dataStore := &mockStore{data: map[string]store.Comment{}}
|
||||
dest := &MockDest{id: 1}
|
||||
dataStore := &mockStore{data: map[string]store.Comment{}}
|
||||
|
||||
dataStore.data["p1"] = store.Comment{ID: "p1"}
|
||||
dataStore.data["p2"] = store.Comment{ID: "p2"}
|
||||
dataStore.data["p1"] = store.Comment{ID: "p1"}
|
||||
dataStore.data["p2"] = store.Comment{ID: "p2"}
|
||||
|
||||
s := NewService(dataStore, 1, dest)
|
||||
assert.NotNil(t, s)
|
||||
s := NewService(dataStore, 1, dest)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
|
||||
synctest.Wait()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
|
||||
synctest.Wait()
|
||||
s.Close()
|
||||
s.Submit(Request{Comment: store.Comment{ID: "c1", ParentID: "p1"}})
|
||||
time.Sleep(time.Millisecond * 110)
|
||||
s.Submit(Request{Comment: store.Comment{ID: "c11", ParentID: "p11"}})
|
||||
time.Sleep(time.Millisecond * 110)
|
||||
s.Close()
|
||||
|
||||
destRes := dest.Get()
|
||||
require.Equal(t, 2, len(destRes), "two comment notified")
|
||||
assert.Equal(t, "p1", destRes[0].Comment.ParentID)
|
||||
assert.Equal(t, "p1", destRes[0].parent.ID)
|
||||
assert.Equal(t, "p11", destRes[1].Comment.ParentID)
|
||||
assert.Equal(t, "", destRes[1].parent.ID)
|
||||
})
|
||||
}
|
||||
|
||||
func TestService_EmailRetrieval(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dest := &MockDest{id: 1}
|
||||
dataStore := &mockStore{data: map[string]store.Comment{}, userDetails: map[string]string{}}
|
||||
|
||||
dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
|
||||
dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u1"}}
|
||||
dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p1", User: store.User{ID: "u2"}}
|
||||
dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
|
||||
dataStore.userDetails["u1"] = "u1@example.com"
|
||||
|
||||
s := NewService(dataStore, 1, dest)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
// one comment, one notification
|
||||
s.Submit(Request{Comment: dataStore.data["p1"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes := dest.Get()
|
||||
require.Equal(t, 1, len(destRes), "one comment notified")
|
||||
assert.Equal(t, "p1", destRes[0].Comment.ID)
|
||||
assert.Empty(t, destRes[0].parent)
|
||||
assert.Empty(t, destRes[0].Emails)
|
||||
|
||||
// reply to the first comment, same comment as one in original comment
|
||||
s.Submit(Request{Comment: dataStore.data["p2"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 2, len(destRes), "two comment notified")
|
||||
assert.Equal(t, "p2", destRes[1].Comment.ID)
|
||||
assert.Equal(t, "p1", destRes[1].parent.ID)
|
||||
assert.Equal(t, "u1", destRes[1].parent.User.ID)
|
||||
assert.Empty(t, destRes[1].Emails, "u1 is not notified they are the one who left the comment")
|
||||
|
||||
// another reply to the first comment, another user
|
||||
s.Submit(Request{Comment: dataStore.data["p3"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 3, len(destRes), "three comment notified")
|
||||
assert.Equal(t, "p3", destRes[2].Comment.ID)
|
||||
assert.Equal(t, "p1", destRes[2].parent.ID)
|
||||
assert.Equal(t, "u1", destRes[2].parent.User.ID)
|
||||
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
|
||||
|
||||
// reply to the last comment by another user, should trigger email retrieval error
|
||||
s.Submit(Request{Comment: dataStore.data["p4"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 4, len(destRes), "four comment notified")
|
||||
assert.Equal(t, "p4", destRes[3].Comment.ID)
|
||||
assert.Equal(t, "p3", destRes[3].parent.ID)
|
||||
assert.Equal(t, "u2", destRes[3].parent.User.ID)
|
||||
assert.Empty(t, destRes[3].Emails, "no email can be retrieved for u2")
|
||||
|
||||
s.Close()
|
||||
})
|
||||
}
|
||||
|
||||
func TestService_Recursive(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dest := &MockDest{id: 1}
|
||||
dataStore := &mockStore{data: map[string]store.Comment{}, userDetails: map[string]string{}}
|
||||
|
||||
dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
|
||||
dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u2"}}
|
||||
dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p2", User: store.User{ID: "u3"}}
|
||||
dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
|
||||
dataStore.data["p5"] = store.Comment{ID: "p5", ParentID: "p4", User: store.User{ID: "u4"}}
|
||||
dataStore.userDetails["u1"] = "u1@example.com"
|
||||
// second comment goes without email address for notification
|
||||
dataStore.userDetails["u3"] = "u3@example.com"
|
||||
|
||||
s := NewService(dataStore, 1, dest)
|
||||
assert.NotNil(t, s)
|
||||
|
||||
// one comment from u1 with email set
|
||||
s.Submit(Request{Comment: dataStore.data["p1"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes := dest.Get()
|
||||
require.Equal(t, 1, len(destRes), "one comment notified")
|
||||
assert.Equal(t, "p1", destRes[0].Comment.ID)
|
||||
assert.Empty(t, destRes[0].parent)
|
||||
assert.Empty(t, destRes[0].Emails)
|
||||
|
||||
// reply to the first comment from u2 without email set
|
||||
s.Submit(Request{Comment: dataStore.data["p2"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 2, len(destRes), "two comment notified")
|
||||
assert.Equal(t, "p2", destRes[1].Comment.ID)
|
||||
assert.Equal(t, "p1", destRes[1].parent.ID)
|
||||
assert.Equal(t, "u1", destRes[1].parent.User.ID)
|
||||
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[1].Emails)
|
||||
|
||||
// reply to the second comment from u3 with email set
|
||||
s.Submit(Request{Comment: dataStore.data["p3"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 3, len(destRes), "three comment notified")
|
||||
assert.Equal(t, "p3", destRes[2].Comment.ID)
|
||||
assert.Equal(t, "p2", destRes[2].parent.ID)
|
||||
assert.Equal(t, "u2", destRes[2].parent.User.ID)
|
||||
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
|
||||
|
||||
// reply to the third comment from u1 (author of the first comment), only u3 should be notified
|
||||
s.Submit(Request{Comment: dataStore.data["p4"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 4, len(destRes), "four comment notified once each")
|
||||
assert.Equal(t, "p4", destRes[3].Comment.ID)
|
||||
assert.Equal(t, "p3", destRes[3].parent.ID)
|
||||
assert.Equal(t, "u3", destRes[3].parent.User.ID)
|
||||
assert.ElementsMatch(t, []string{"u3@example.com"}, destRes[3].Emails, "u1 is not notified they are the one who left the comment")
|
||||
|
||||
// reply to the fourth comment from u4, u1 and u3 should be notified once as a result
|
||||
s.Submit(Request{Comment: dataStore.data["p5"]})
|
||||
synctest.Wait()
|
||||
|
||||
destRes = dest.Get()
|
||||
require.Equal(t, 5, len(destRes), "four comment notified once each")
|
||||
assert.Equal(t, "p5", destRes[4].Comment.ID)
|
||||
assert.Equal(t, "p4", destRes[4].parent.ID)
|
||||
assert.Equal(t, "u1", destRes[4].parent.User.ID)
|
||||
assert.ElementsMatch(t, []string{"u1@example.com", "u3@example.com"}, destRes[4].Emails, "u3 and u1 notified once")
|
||||
|
||||
s.Close()
|
||||
})
|
||||
destRes := dest.Get()
|
||||
require.Equal(t, 2, len(destRes), "two comment notified")
|
||||
assert.Equal(t, "p1", destRes[0].Comment.ParentID)
|
||||
assert.Equal(t, "p1", destRes[0].parent.ID)
|
||||
assert.Equal(t, "p11", destRes[1].Comment.ParentID)
|
||||
assert.Equal(t, "", destRes[1].parent.ID)
|
||||
}
|
||||
|
||||
func TestService_Nop(t *testing.T) {
|
||||
s := NopService
|
||||
s.Submit(Request{Comment: store.Comment{}})
|
||||
s.Close()
|
||||
assert.Equal(t, uint32(1), s.closed.Load())
|
||||
assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
|
||||
}
|
||||
|
||||
type mockStore struct {
|
||||
data map[string]store.Comment
|
||||
userDetails map[string]string
|
||||
}
|
||||
|
||||
func (m mockStore) getUserDetail(userID string) (string, error) {
|
||||
detail, ok := m.userDetails[userID]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no such user")
|
||||
}
|
||||
return detail, nil
|
||||
}
|
||||
type mockStore struct{ data map[string]store.Comment }
|
||||
|
||||
func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
|
||||
res, ok := m.data[id]
|
||||
if !ok {
|
||||
return store.Comment{}, fmt.Errorf("no such id")
|
||||
return store.Comment{}, errors.New("no such id")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m mockStore) GetUserEmail(_, userID string) (string, error) {
|
||||
return m.getUserDetail(userID)
|
||||
}
|
||||
|
||||
func (m mockStore) GetUserTelegram(_, userID string) (string, error) {
|
||||
return m.getUserDetail(userID)
|
||||
func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
|
||||
return "", errors.New("no such user")
|
||||
}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// pruneHTML prunes string keeping HTML closing tags.
|
||||
// maxLength applies to visible text only, not HTML tags.
|
||||
func pruneHTML(htmlText string, maxLength int) string {
|
||||
var result strings.Builder
|
||||
var endTokens []string
|
||||
visibleLen := 0
|
||||
|
||||
suffix := "..."
|
||||
suffixLen := len(suffix)
|
||||
|
||||
tokenizer := html.NewTokenizer(strings.NewReader(htmlText))
|
||||
for {
|
||||
if tokenizer.Next() == html.ErrorToken {
|
||||
return result.String()
|
||||
}
|
||||
token := tokenizer.Token()
|
||||
|
||||
switch token.Type {
|
||||
case html.CommentToken, html.DoctypeToken:
|
||||
continue
|
||||
|
||||
case html.StartTagToken:
|
||||
endTokens = append([]string{fmt.Sprintf("</%s>", token.Data)}, endTokens...)
|
||||
result.WriteString(token.String())
|
||||
|
||||
case html.EndTagToken:
|
||||
if len(endTokens) > 0 {
|
||||
endTokens = endTokens[1:]
|
||||
}
|
||||
result.WriteString(token.String())
|
||||
|
||||
case html.SelfClosingTagToken:
|
||||
result.WriteString(token.String())
|
||||
|
||||
case html.TextToken:
|
||||
text := token.String()
|
||||
if visibleLen+len(text)+suffixLen > maxLength {
|
||||
remaining := maxLength - visibleLen - suffixLen
|
||||
text = pruneStringToWord(text, remaining)
|
||||
result.WriteString(text)
|
||||
result.WriteString(suffix)
|
||||
for _, endTag := range endTokens {
|
||||
result.WriteString(endTag)
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
visibleLen += len(text)
|
||||
result.WriteString(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pruneStringToWord prunes string to specified length respecting word boundaries
|
||||
func pruneStringToWord(text string, maxLength int) string {
|
||||
if maxLength <= 0 {
|
||||
return ""
|
||||
}
|
||||
if len(text) <= maxLength {
|
||||
return text
|
||||
}
|
||||
|
||||
// find last space at or before maxLength to cut at word boundary
|
||||
lastSpace := strings.LastIndex(text[:maxLength+1], " ")
|
||||
if lastSpace <= 0 {
|
||||
return ""
|
||||
}
|
||||
return text[:lastSpace]
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestPruneHTML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
html string
|
||||
maxLength int
|
||||
expected string
|
||||
}{
|
||||
{"within limit", "<p>Hello</p>", 20, "<p>Hello</p>"},
|
||||
{"exceeds limit", "<p>Hello world, this is a long text</p>", 15, "<p>Hello world,...</p>"},
|
||||
{"nested tags", "<div><p>Hello world</p><p>More text</p></div>", 20, "<div><p>Hello world</p><p>More...</p></div>"},
|
||||
{"html comment stripped", "<!-- comment --><p>Hello</p>", 20, "<p>Hello</p>"},
|
||||
{"self-closing tag", "<p>Hello<br/>World</p>", 8, "<p>Hello<br/>...</p>"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, pruneHTML(tt.html, tt.maxLength))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneStringToWord(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
maxLength int
|
||||
expected string
|
||||
}{
|
||||
{"within limit", "hello world", 15, "hello world"},
|
||||
{"cut at word boundary", "hello world and more", 11, "hello world"},
|
||||
{"zero length", "hello", 0, ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, pruneStringToWord(tt.text, tt.maxLength))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
)
|
||||
|
||||
// Slack implements notify.Destination for Slack
|
||||
type Slack struct {
|
||||
*ntf.Slack
|
||||
|
||||
channelName string
|
||||
}
|
||||
|
||||
// NewSlack makes Slack bot for notifications
|
||||
func NewSlack(token, channelName string) *Slack {
|
||||
log.Printf("[DEBUG] create new slack notifier for chan %s", channelName)
|
||||
if channelName == "" {
|
||||
channelName = "general"
|
||||
}
|
||||
|
||||
return &Slack{Slack: ntf.NewSlack(token), channelName: channelName}
|
||||
}
|
||||
|
||||
// Send to Slack channel
|
||||
func (s *Slack) Send(ctx context.Context, req Request) error {
|
||||
log.Printf("[DEBUG] send slack notification, comment id %s", req.Comment.ID)
|
||||
|
||||
user := req.Comment.User.Name
|
||||
if req.Comment.ParentID != "" {
|
||||
user += " → " + req.parent.User.Name
|
||||
}
|
||||
|
||||
title := "↦ original comment"
|
||||
if req.Comment.PostTitle != "" {
|
||||
title = "↦ " + req.Comment.PostTitle
|
||||
}
|
||||
|
||||
destination := fmt.Sprintf(
|
||||
"slack:%s?title=%s&attachmentText=%s&titleLink=%s",
|
||||
s.channelName,
|
||||
url.QueryEscape(title),
|
||||
url.QueryEscape(req.Comment.Orig),
|
||||
url.QueryEscape(req.Comment.Locator.URL+uiNav+req.Comment.ID),
|
||||
)
|
||||
|
||||
return s.Slack.Send(ctx, destination, "New comment from "+user)
|
||||
}
|
||||
|
||||
// SendVerification is not implemented for Slack
|
||||
func (s *Slack) SendVerification(_ context.Context, _ VerificationRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Slack) String() string {
|
||||
return s.Slack.String() + " for channel " + s.channelName + ""
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
)
|
||||
|
||||
func TestSlack_New(t *testing.T) {
|
||||
ts := NewSlack("", "")
|
||||
assert.NotNil(t, ts)
|
||||
assert.Equal(t, "general", ts.channelName)
|
||||
}
|
||||
|
||||
func TestSlack_Send(t *testing.T) {
|
||||
ts := NewSlack("", "")
|
||||
|
||||
c := store.Comment{PostTitle: "test title", Text: "some text", ParentID: "1", ID: "999"}
|
||||
c.User.Name = "from"
|
||||
cp := store.Comment{Text: "some parent text"}
|
||||
cp.User.Name = "to"
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := ts.Send(ctx, Request{Comment: c, parent: cp})
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSlack_Name(t *testing.T) {
|
||||
tb := NewSlack("", "test-channel")
|
||||
assert.Equal(t, "slack notifications destination for channel test-channel", tb.String())
|
||||
}
|
||||
|
||||
func TestSlack_SendVerification(t *testing.T) {
|
||||
ts := NewSlack("", "")
|
||||
assert.NoError(t, ts.SendVerification(context.Background(), VerificationRequest{}))
|
||||
}
|
||||
+120
-80
@@ -1,117 +1,157 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/go-pkgz/repeater"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const commentTextLengthLimit = 100
|
||||
|
||||
// TelegramParams contain settings for telegram notifications
|
||||
type TelegramParams struct {
|
||||
AdminChannelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
|
||||
Token string // token for telegram bot API interactions
|
||||
Timeout time.Duration // http client timeout
|
||||
UserNotifications bool // flag which enables user notifications
|
||||
ErrorMsg, SuccessMsg string // messages for successful and unsuccessful subscription requests to bot
|
||||
}
|
||||
|
||||
// Telegram implements notify.Destination for telegram
|
||||
type Telegram struct {
|
||||
*ntf.Telegram
|
||||
|
||||
AdminChannelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
|
||||
UserNotifications bool // flag which enables user notifications
|
||||
channelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
|
||||
token string
|
||||
apiPrefix string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
const telegramTimeOut = 5000 * time.Millisecond
|
||||
const telegramAPIPrefix = "https://api.telegram.org/bot"
|
||||
|
||||
// NewTelegram makes telegram bot for notifications
|
||||
func NewTelegram(params TelegramParams) (*Telegram, error) {
|
||||
client, err := ntf.NewTelegram(ntf.TelegramParams{
|
||||
Token: params.Token,
|
||||
Timeout: params.Timeout,
|
||||
ErrorMsg: params.ErrorMsg,
|
||||
SuccessMsg: params.SuccessMsg,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
|
||||
|
||||
if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
|
||||
channelID = "@" + channelID // if channelID not a number enforce @ prefix
|
||||
}
|
||||
|
||||
return &Telegram{Telegram: client, AdminChannelID: params.AdminChannelID, UserNotifications: params.UserNotifications}, nil
|
||||
}
|
||||
res := Telegram{channelID: channelID, token: token, apiPrefix: api, timeout: timeout}
|
||||
if res.apiPrefix == "" {
|
||||
res.apiPrefix = telegramAPIPrefix
|
||||
}
|
||||
if res.timeout == 0 {
|
||||
res.timeout = telegramTimeOut
|
||||
}
|
||||
log.Printf("[DEBUG] create new telegram notifier for chan %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
|
||||
|
||||
// Send to telegram recipients
|
||||
func (t *Telegram) Send(ctx context.Context, req Request) error {
|
||||
log.Printf("[DEBUG] send telegram notification for comment ID %s", req.Comment.ID)
|
||||
var errs []error
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
msg := t.buildMessage(req)
|
||||
|
||||
if t.AdminChannelID != "" {
|
||||
err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", t.AdminChannelID), msg)
|
||||
err := repeater.NewDefault(5, time.Millisecond*250).Do(ctx, func() error {
|
||||
client := http.Client{Timeout: telegramTimeOut}
|
||||
resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
|
||||
if err != nil {
|
||||
errs = append(errs,
|
||||
fmt.Errorf("problem sending admin telegram notification about comment ID %s to %s: %w",
|
||||
req.Comment.ID, t.AdminChannelID, err,
|
||||
),
|
||||
)
|
||||
return errors.Wrap(err, "can't initialize telegram notifications")
|
||||
}
|
||||
}
|
||||
|
||||
if t.UserNotifications {
|
||||
for _, user := range req.Telegrams {
|
||||
err := t.Telegram.Send(ctx, fmt.Sprintf("telegram:%s?parseMode=HTML", user), msg)
|
||||
if err != nil {
|
||||
errs = append(errs,
|
||||
fmt.Errorf("problem sending user telegram notification about comment ID %s to %q: %w",
|
||||
req.Comment.ID, user, err,
|
||||
),
|
||||
)
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close request body, %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.Errorf("unexpected telegram status code %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
|
||||
tgResp := struct {
|
||||
OK bool `json:"ok"`
|
||||
Result struct {
|
||||
FirstName string `json:"first_name"`
|
||||
ID uint64 `json:"id"`
|
||||
IsBot bool `json:"is_bot"`
|
||||
UserName string `json:"username"`
|
||||
}
|
||||
}{}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
|
||||
return errors.Wrap(err, "can't decode response")
|
||||
}
|
||||
|
||||
if !tgResp.OK || !tgResp.Result.IsBot {
|
||||
return errors.Errorf("unexpected telegram response %+v", tgResp)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return &res, err
|
||||
}
|
||||
|
||||
// buildMessage generates message for generic notification about new comment
|
||||
func (t *Telegram) buildMessage(req Request) string {
|
||||
commentURLPrefix := req.Comment.Locator.URL + uiNav
|
||||
|
||||
msg := fmt.Sprintf(`<a href=%q>%s</a>`, commentURLPrefix+req.Comment.ID, ntf.EscapeTelegramText(req.Comment.User.Name))
|
||||
|
||||
if req.Comment.ParentID != "" {
|
||||
msg += fmt.Sprintf(" -> <a href=%q>%s</a>", commentURLPrefix+req.parent.ID, ntf.EscapeTelegramText(req.parent.User.Name))
|
||||
// Send to telegram channel
|
||||
func (t *Telegram) Send(ctx context.Context, req Request) error {
|
||||
if req.Comment.ID == "" {
|
||||
// verification request received, send nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
msg += fmt.Sprintf("\n\n%s", pruneHTML(ntf.TelegramSupportedHTML(req.Comment.Text), commentTextLengthLimit))
|
||||
|
||||
if req.Comment.ParentID != "" {
|
||||
msg += fmt.Sprintf("\n\n\"<i>%s</i>\"", pruneHTML(ntf.TelegramSupportedHTML(req.parent.Text), commentTextLengthLimit))
|
||||
if req.ForAdmin {
|
||||
// request for administrator received, do nothing with it
|
||||
// as we already sent message on request without this flag set
|
||||
return nil
|
||||
}
|
||||
client := http.Client{Timeout: telegramTimeOut}
|
||||
log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
|
||||
|
||||
from := req.Comment.User.Name
|
||||
if req.Comment.ParentID != "" {
|
||||
from += " → " + req.parent.User.Name
|
||||
}
|
||||
from = "*" + from + "*"
|
||||
link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
|
||||
if req.Comment.PostTitle != "" {
|
||||
msg += fmt.Sprintf("\n\n↦ <a href=%q>%s</a>", req.Comment.Locator.URL, ntf.EscapeTelegramText(req.Comment.PostTitle))
|
||||
link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
|
||||
}
|
||||
u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
|
||||
t.apiPrefix, t.token, t.channelID)
|
||||
|
||||
msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.Comment.Orig, link)
|
||||
msg = html.UnescapeString(msg)
|
||||
body := struct {
|
||||
Text string `json:"text"`
|
||||
}{Text: msg}
|
||||
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to make telegram body")
|
||||
}
|
||||
|
||||
return msg
|
||||
}
|
||||
r, err := http.NewRequest("POST", u, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to make telegram request")
|
||||
}
|
||||
r.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
// SendVerification is not needed for telegram
|
||||
func (t *Telegram) SendVerification(_ context.Context, _ VerificationRequest) error {
|
||||
r = r.WithContext(ctx)
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get telegram response")
|
||||
}
|
||||
defer func() {
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close request body, %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.Errorf("unexpected telegram status code %d for url %q", resp.StatusCode, u)
|
||||
}
|
||||
|
||||
tgResp := struct {
|
||||
OK bool `json:"ok"`
|
||||
}{}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
|
||||
return errors.Wrap(err, "can't decode telegram response")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Telegram) String() string {
|
||||
result := t.Telegram.String()
|
||||
if t.AdminChannelID != "" {
|
||||
result += " with admin notifications to " + t.AdminChannelID
|
||||
}
|
||||
if t.UserNotifications {
|
||||
result += " with user notifications enabled"
|
||||
}
|
||||
return result
|
||||
return "telegram: " + t.channelID
|
||||
}
|
||||
|
||||
@@ -2,69 +2,124 @@ package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
"github.com/umputun/remark/backend/app/store"
|
||||
)
|
||||
|
||||
func TestTelegram_NewError(t *testing.T) {
|
||||
tb, err := NewTelegram(TelegramParams{})
|
||||
func TestTelegram_New(t *testing.T) {
|
||||
|
||||
ts := mockTelegramServer()
|
||||
defer ts.Close()
|
||||
|
||||
tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, tb)
|
||||
assert.Equal(t, "@remark_test", tb.channelID, "@ added")
|
||||
|
||||
st := time.Now()
|
||||
_, err = NewTelegram("bad-resp", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.EqualError(t, err, "unexpected telegram response {OK:false Result:{FirstName:comments_test ID:707381019 IsBot:false UserName:remark42_test_bot}}")
|
||||
assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
|
||||
|
||||
_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, tb)
|
||||
assert.Contains(t, err.Error(), "can't decode response:")
|
||||
|
||||
_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.EqualError(t, err, "unexpected telegram status code 404")
|
||||
|
||||
_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "can't initialize telegram notifications")
|
||||
assert.Contains(t, err.Error(), "dial tcp 127.0.0.1:4321: connect: connection refused")
|
||||
|
||||
_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
|
||||
assert.Error(t, err, "empty api url not allowed")
|
||||
|
||||
_, err = NewTelegram("good-token", "remark_test", 0, ts.URL+"/")
|
||||
assert.NoError(t, err, "0 timeout allowed as default")
|
||||
|
||||
tb, err = NewTelegram("good-token", "1234567890", 2*time.Second, ts.URL+"/")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, tb)
|
||||
assert.Equal(t, "1234567890", tb.channelID, "no @ prefix")
|
||||
}
|
||||
|
||||
func TestTelegram_Send(t *testing.T) {
|
||||
tb := Telegram{
|
||||
AdminChannelID: "remark_test",
|
||||
UserNotifications: true,
|
||||
Telegram: &ntf.Telegram{}, // broken sender due to unset API
|
||||
}
|
||||
assert.Equal(t, "telegram notifications destination with admin notifications to remark_test with user notifications enabled", tb.String())
|
||||
c := store.Comment{Text: "some text", ParentID: "1", ID: "999", PostTitle: "[test title]", Locator: store.Locator{URL: "http://example.org/"}}
|
||||
ts := mockTelegramServer()
|
||||
defer ts.Close()
|
||||
|
||||
tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, tb)
|
||||
c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
|
||||
c.User.Name = "from"
|
||||
cp := store.Comment{Text: `<p>some parent text with a <a href="http://example.org">link</a> and special text:<br>& < > &</p>`}
|
||||
cp := store.Comment{Text: "some parent text"}
|
||||
cp.User.Name = "to"
|
||||
|
||||
err := tb.Send(context.Background(), Request{Comment: c, parent: cp, Telegrams: []string{"test_user_channel"}})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "problem sending user telegram notification about comment ID 999 to \"test_user_channel\"")
|
||||
assert.Contains(t, err.Error(), "problem sending admin telegram notification about comment ID 999 to remark_test")
|
||||
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
|
||||
assert.NoError(t, err)
|
||||
c.PostTitle = "test title"
|
||||
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// test buildMessage separately for message text
|
||||
res := tb.buildMessage(Request{Comment: c, parent: cp})
|
||||
assert.Equal(t, `<a href="http://example.org/#remark42__comment-999">from</a> -> <a href="http://example.org/#remark42__comment-">to</a>
|
||||
tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
|
||||
assert.Error(t, err, "should failed")
|
||||
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
|
||||
|
||||
some text
|
||||
|
||||
"<i>some parent text with a <a href="http://example.org">link</a> and special text:& < > &</i>"
|
||||
|
||||
↦ <a href="http://example.org/">[test title]</a>`,
|
||||
res)
|
||||
|
||||
// special case for text with h1-h6 header
|
||||
ch := store.Comment{Text: "<h1>Hello</h1><h6>World</h6>", ID: "555", Locator: store.Locator{URL: "http://example.org/"}}
|
||||
ch.User.Name = "from"
|
||||
res = tb.buildMessage(Request{Comment: ch})
|
||||
assert.Equal(t, `<a href="http://example.org/#remark42__comment-555">from</a>
|
||||
|
||||
<b>Hello</b><i><b>World</b></i>`,
|
||||
res)
|
||||
|
||||
// prune string keeping HTML closing tags
|
||||
c = store.Comment{
|
||||
Text: "<b>Lorem ipsum <i>dolor sit amet</i>, consectetur adipiscing <code>elit, sed do eiusmod tempor incididunt</code> ut labore et dolore magna aliqua.</b>",
|
||||
}
|
||||
res = tb.buildMessage(Request{Comment: c})
|
||||
assert.Equal(t, `<a href="#remark42__comment-"></a>
|
||||
|
||||
<b>Lorem ipsum <i>dolor sit amet</i>, consectetur adipiscing <code>elit, sed do eiusmod tempor incididunt</code> ut...</b>`, res)
|
||||
assert.Equal(t, "telegram: @remark_test", tb.String())
|
||||
require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
|
||||
}
|
||||
|
||||
func TestTelegram_SendVerification(t *testing.T) {
|
||||
tb := Telegram{}
|
||||
// empty VerificationRequest should return no error and do nothing, as well as any other
|
||||
assert.NoError(t, tb.SendVerification(context.Background(), VerificationRequest{}))
|
||||
func mockTelegramServer() *httptest.Server {
|
||||
router := chi.NewRouter()
|
||||
router.Get("/good-token/getMe", func(w http.ResponseWriter, r *http.Request) {
|
||||
s := `{"ok": true,
|
||||
"result": {
|
||||
"first_name": "comments_test",
|
||||
"id": 707381019,
|
||||
"is_bot": true,
|
||||
"username": "remark42_test_bot"
|
||||
}}`
|
||||
_, _ = w.Write([]byte(s))
|
||||
})
|
||||
router.Get("/bad-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
|
||||
s := `{"ok": false,
|
||||
"result": {
|
||||
"first_name": "comments_test",
|
||||
"id": 707381019,
|
||||
"is_bot": false,
|
||||
"username": "remark42_test_bot"
|
||||
}}`
|
||||
_, _ = w.Write([]byte(s))
|
||||
})
|
||||
router.Get("/non-json-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
|
||||
s := `"ok": false,
|
||||
"result": {
|
||||
"first_name": "comments_test",
|
||||
"id": 707381019,
|
||||
"is_bot": false,
|
||||
"username": "remark42_test_bot"
|
||||
`
|
||||
_, _ = w.Write([]byte(s))
|
||||
})
|
||||
router.Get("/404/getMe", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(404)
|
||||
})
|
||||
|
||||
router.Post("/good-token/sendMessage", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"ok": true}`))
|
||||
})
|
||||
|
||||
return httptest.NewServer(router)
|
||||
}
|
||||
|
||||
-1
@@ -1 +0,0 @@
|
||||
{{
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
{{- if .ForAdmin}}
|
||||
New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
|
||||
{{- else }}
|
||||
New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
|
||||
{{- end }}
|
||||
{{- if .ParentCommentText}}
|
||||
{{.ParentUserPicture}}
|
||||
{{.ParentUserName}}
|
||||
{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}
|
||||
Parent comment link: {{.ParentCommentLink}}
|
||||
{{.ParentCommentText}}
|
||||
{{- end }}
|
||||
|
||||
User: {{.UserName}}
|
||||
{{.CommentDate.Format "02.01.2006 at 15:04"}}
|
||||
Comment: {{.CommentText}}
|
||||
{{.Email}} {{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}
|
||||
{{- if .UnsubscribeLink}}
|
||||
Unsubscribe link: {{.UnsubscribeLink}}
|
||||
{{- end }}
|
||||
@@ -1,7 +0,0 @@
|
||||
Confirmation for {{.User}} on site {{.Site}}
|
||||
{{- if .SubscribeURL}}
|
||||
Subscribe url: {{.SubscribeURL}}{{.Token}}
|
||||
{{- end }}
|
||||
Token:{{.Token}}
|
||||
Sent to {{.Email}}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
log "github.com/go-pkgz/lgr"
|
||||
ntf "github.com/go-pkgz/notify"
|
||||
)
|
||||
|
||||
const (
|
||||
webhookDefaultTemplate = `{"text": {{.Text | escapeJSONString}}}`
|
||||
)
|
||||
|
||||
// WebhookParams contain settings for webhook notifications
|
||||
type WebhookParams struct {
|
||||
URL string
|
||||
Template string
|
||||
Headers []string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Webhook implements notify.Destination for Webhook notifications
|
||||
type Webhook struct {
|
||||
*ntf.Webhook
|
||||
|
||||
url string
|
||||
template *template.Template
|
||||
}
|
||||
|
||||
// NewWebhook makes Webhook
|
||||
func NewWebhook(params WebhookParams) (*Webhook, error) {
|
||||
res := &Webhook{
|
||||
Webhook: ntf.NewWebhook(ntf.WebhookParams{
|
||||
Timeout: params.Timeout,
|
||||
Headers: params.Headers,
|
||||
}),
|
||||
url: params.URL,
|
||||
}
|
||||
|
||||
if res.url == "" {
|
||||
return nil, fmt.Errorf("webhook URL is required for webhook notifications")
|
||||
}
|
||||
|
||||
if params.Template == "" {
|
||||
params.Template = webhookDefaultTemplate
|
||||
}
|
||||
|
||||
payloadTmpl, err := template.New("webhook").Funcs(template.FuncMap{"escapeJSONString": escapeJSONString}).Parse(params.Template)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse webhook template: %w", err)
|
||||
}
|
||||
|
||||
res.template = payloadTmpl
|
||||
|
||||
log.Printf("[DEBUG] create new webhook notifier for %s", res.url)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Send sends Webhook notification
|
||||
func (w *Webhook) Send(ctx context.Context, req Request) error {
|
||||
log.Printf("[DEBUG] send webhook notification, comment id %s", req.Comment.ID)
|
||||
var payload bytes.Buffer
|
||||
err := w.template.Execute(&payload, req.Comment)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to compile webhook template: %w", err)
|
||||
}
|
||||
|
||||
return w.Webhook.Send(ctx, w.url, payload.String())
|
||||
}
|
||||
|
||||
// SendVerification is not implemented for Webhook
|
||||
func (w *Webhook) SendVerification(_ context.Context, _ VerificationRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// String describes the webhook instance
|
||||
func (w *Webhook) String() string {
|
||||
return fmt.Sprintf("%s to %s", w.Webhook.String(), w.url)
|
||||
}
|
||||
|
||||
// escapeJSONString escapes string for JSON
|
||||
func escapeJSONString(s string) (string, error) {
|
||||
b, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/umputun/remark42/backend/app/store"
|
||||
)
|
||||
|
||||
func TestWebhook_NewWebhook(t *testing.T) {
|
||||
wh, err := NewWebhook(WebhookParams{
|
||||
URL: "https://example.org/webhook",
|
||||
Headers: []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, wh)
|
||||
|
||||
assert.Equal(t, "https://example.org/webhook", wh.url)
|
||||
assert.Equal(t, []string{"Authorization:Basic AXVubzpwQDU1dzByYM=="}, wh.Headers)
|
||||
assert.NotNil(t, wh.template)
|
||||
|
||||
wh, err = NewWebhook(WebhookParams{})
|
||||
assert.Nil(t, wh)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "webhook URL is required for webhook notifications", err.Error())
|
||||
|
||||
wh, err = NewWebhook(WebhookParams{URL: "https://example.org/webhook", Template: "{{.Text"})
|
||||
assert.Nil(t, wh)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unable to parse webhook template")
|
||||
}
|
||||
|
||||
// https://github.com/umputun/remark42/issues/1791
|
||||
func TestWebhook_ReceiveValidJSON(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, r.URL.Path, "/webhook-notify")
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
t.Log("received body", string(body))
|
||||
assert.JSONEq(t, `{"text": "<p>testme</p>\n"}`, string(body))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
wh, err := NewWebhook(WebhookParams{
|
||||
URL: ts.URL + "/webhook-notify",
|
||||
Headers: []string{"Content-Type:application/json,text/plain"},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, wh)
|
||||
|
||||
f := store.NewCommentFormatter()
|
||||
c := store.Comment{Text: f.FormatText("testme", false), ParentID: "1", ID: "999"}
|
||||
|
||||
err = wh.Send(context.Background(), Request{Comment: c})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestWebhook_Send(t *testing.T) {
|
||||
wh, err := NewWebhook(WebhookParams{
|
||||
URL: "bad-url",
|
||||
Headers: []string{"Content-Type:application/json,text/plain", ""},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, wh)
|
||||
|
||||
c := store.Comment{Text: "some text", ParentID: "1", ID: "999"}
|
||||
c.User.Name = "from"
|
||||
|
||||
err = wh.Send(context.Background(), Request{Comment: c})
|
||||
assert.Error(t, err)
|
||||
|
||||
wh, err = NewWebhook(WebhookParams{
|
||||
URL: "https://example.org/webhook",
|
||||
Template: "{{.InvalidProperty}}",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
err = wh.Send(context.Background(), Request{Comment: c})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "webhook template")
|
||||
|
||||
wh, err = NewWebhook(WebhookParams{URL: "https://example.org/webhook"})
|
||||
assert.NoError(t, err)
|
||||
err = wh.Send(nil, Request{Comment: c}) // nolint
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unable to create webhook request")
|
||||
|
||||
wh, err = NewWebhook(WebhookParams{URL: "https://not-existing-url.net"})
|
||||
assert.NoError(t, err)
|
||||
err = wh.Send(context.Background(), Request{Comment: c})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "webhook request failed")
|
||||
}
|
||||
|
||||
func TestWebhook_SendVerification(t *testing.T) {
|
||||
wh, err := NewWebhook(WebhookParams{URL: "https://example.org/webhook"})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, wh)
|
||||
|
||||
err = wh.SendVerification(context.Background(), VerificationRequest{})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestWebhook_String(t *testing.T) {
|
||||
wh, err := NewWebhook(WebhookParams{URL: "https://example.org/webhook", Timeout: time.Minute * 5})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, wh)
|
||||
|
||||
str := wh.String()
|
||||
assert.Equal(t, "webhook notification with timeout 5m0s to https://example.org/webhook", str)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user