Compare commits

..
Author SHA1 Message Date
Umputun dfb3436f30 backport url sanitizer to 1.6 2021-03-26 16:42:56 -05:00
1227 changed files with 122781 additions and 120935 deletions
-2
View File
@@ -85,8 +85,6 @@ steps:
commands:
- ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
- ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
- ssh umputun@remark42.com "cd /srv/remark-site && git pull && git submodule update --recursive --remote"
- ssh umputun@remark42.com "cd /srv/remark-site && docker-compose build && docker-compose up"
when:
branch: master
event: push
@@ -1,19 +0,0 @@
name: frontend
on:
pull_request:
paths:
- '.github/workflows/ci-frontend-size-limit.yml'
- 'frontend/**'
jobs:
size:
runs-on: ubuntu-latest
env:
CI_JOB_NUMBER: 1
steps:
- uses: actions/checkout@v1
- uses: andresz1/size-limit-action@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
directory: frontend
-105
View File
@@ -1,105 +0,0 @@
name: frontend
on:
push:
branches:
tags:
paths:
- '.github/workflows/ci-frontend.yml'
- 'frontend/**'
pull_request:
paths:
- '.github/workflows/ci-frontend.yml'
- 'frontend/**'
jobs:
check-transtations:
runs-on: ubuntu-latest
strategy:
matrix:
node: [14.15]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- run: npm ci --loglevel warn
working-directory: ./frontend
- uses: actions/cache@v2
with:
path: ${{ github.workspace }}/frontend/node_modules/.cache
key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- run: npm run check:translation
working-directory: ./frontend
check-typescript:
runs-on: ubuntu-latest
strategy:
matrix:
node: [14.15]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- run: npm ci --loglevel warn
working-directory: ./frontend
- uses: actions/cache@v2
with:
path: ${{ github.workspace }}/frontend/node_modules/.cache
key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- run: npm run check:types
working-directory: ./frontend
lint:
runs-on: ubuntu-latest
strategy:
matrix:
node: [14.15]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- run: npm ci --loglevel warn
working-directory: ./frontend
- run: npx run-p lint
working-directory: ./frontend
test:
runs-on: ubuntu-latest
strategy:
matrix:
node: [14.15]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- run: npm ci --loglevel warn
working-directory: ./frontend
- run: npm run test:coverage
working-directory: ./frontend
- name: submit coverage
run: node ${{ github.workspace }}/frontend/node_modules/.bin/codecov
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: install golangci-lint and goveralls
run: |
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.26.0
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.25.0
go get -u github.com/mattn/goveralls
- name: test and lint backend
+28
View File
@@ -0,0 +1,28 @@
name: test_frontend
on:
push:
branches:
tags:
paths:
- '.github/workflows/ci-test-frontend.yml'
- 'frontend/**'
pull_request:
paths:
- '.github/workflows/ci-test-frontend.yml'
- 'frontend/**'
jobs:
frontend:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
- run: npm ci --loglevel warn
working-directory: ./frontend
- run: npx run-p check lint
working-directory: ./frontend
+5 -9
View File
@@ -1,4 +1,4 @@
FROM umputun/baseimage:buildgo-v1.6.1 as build-backend
FROM umputun/baseimage:buildgo-latest as build-backend
ARG CI
ARG DRONE
@@ -16,14 +16,11 @@ WORKDIR /build/backend
ENV GOFLAGS="-mod=vendor"
# install gcc in order to be able to go test package with -race
RUN apk --no-cache add gcc libc-dev
# run tests
RUN \
cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then \
CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
golangci-lint run --config ../.golangci.yml ./... ; \
else echo "skip backend tests and linter" ; fi
@@ -35,7 +32,7 @@ RUN \
echo "version=$version" && \
go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
FROM node:12.16-alpine as build-frontend-deps
FROM node:10.11-alpine as build-frontend-deps
ARG CI
ENV HUSKY_SKIP_INSTALL=true
@@ -45,7 +42,7 @@ ADD frontend/package.json /srv/frontend/package.json
ADD frontend/package-lock.json /srv/frontend/package-lock.json
RUN cd /srv/frontend && CI=true npm ci --loglevel warn
FROM node:12.16-alpine as build-frontend
FROM node:10.11-alpine as build-frontend
ARG CI
ARG SKIP_FRONTEND_TEST
@@ -58,7 +55,7 @@ RUN cd /srv/frontend && \
else echo "skip frontend tests and lint" ; npm run build ; fi && \
rm -rf ./node_modules
FROM umputun/baseimage:app-v1.6.1
FROM umputun/baseimage:app
WORKDIR /srv
@@ -69,7 +66,6 @@ ADD backend/scripts/import.sh /usr/local/bin/import
RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
COPY --from=build-backend /build/backend/remark42 /srv/remark42
COPY --from=build-backend /build/backend/templates /srv
COPY --from=build-frontend /srv/frontend/public/ /srv/web
RUN chown -R app:app /srv
RUN ln -s /srv/remark42 /usr/bin/remark42
+5 -5
View File
@@ -1,4 +1,4 @@
FROM node:12.16-alpine as build-frontend-deps
FROM node:10.11-alpine as build-frontend-deps
ARG CI
ARG DRONE
@@ -13,7 +13,7 @@ ADD frontend/package.json /srv/frontend/package.json
ADD frontend/package-lock.json /srv/frontend/package-lock.json
RUN cd /srv/frontend && CI=true npm ci
FROM node:12.16-alpine as build-frontend
FROM node:10.11-alpine as build-frontend
ARG CI
ARG NODE_ENV=production
@@ -31,6 +31,8 @@ FROM umputun/baseimage:buildgo-latest as build-backend
ARG GITHUB_TOKEN
ENV SKIP_BACKEND_TEST=true
RUN go get github.com/rakyll/statik
WORKDIR /build/backend
ADD backend /build/backend
ADD README.md /build/
@@ -42,10 +44,8 @@ COPY --from=build-frontend /srv/frontend/public/ web
RUN \
export WEB_ROOT=/build/backend/web && \
find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \
sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \
ls -la /build/backend/app/templates/statik.go && \
ls -la /build/backend/app/rest/api/statik.go && \
ls -la /build/backend/web/
-1
View File
@@ -34,7 +34,6 @@ frontend:
docker-compose -f compose-dev-frontend.yml build
rundev:
docker pull umputun/baseimage:buildgo-latest
SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
docker-compose -f compose-private.yml up
+44 -53
View File
@@ -1,11 +1,10 @@
# remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://codecov.io/gh/umputun/remark42)
# remark42 [![Build Status](https://github.com/umputun/remark/workflows/build/badge.svg)](https://github.com/umputun/remark/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
* Social login via Google, Twitter, Facebook, Microsoft, GitHub and Yandex
* Social login via Google, Twitter, Facebook, GitHub and Yandex
* Login via email
* Optional anonymous access
* Multi-level nested comments with both tree and plain presentations
@@ -32,9 +31,9 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
<details><summary>Screenshots</summary>
Comments example:
![](https://github.com/umputun/remark42/blob/master/screenshots/comments.png)
![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark42/wiki/Admin-UI)
For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
</details>
@@ -95,7 +94,7 @@ _this is the recommended way to run remark42_
#### Without Docker
* download archive for [stable release](https://github.com/umputun/remark42/releases) or [development version](https://remark42.com/downloads)
* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
@@ -110,12 +109,10 @@ _this is the recommended way to run remark42_
| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `rpc` |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| admin.shared.id | ADMIN_SHARED_ID | | admin ids (list of user ids), _multi_ |
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin emails, _multi_ |
| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ |
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| cache.type | CACHE_TYPE | `mem` | type of cache, `redis_pub_sub` or `mem` or `none` |
| cache.redis_addr | CACHE_REDIS_ADDR | `127.0.0.1:6379` | address of redis PubSub instance, turn `redis_pub_sub` cache on for distributed cache |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
@@ -134,16 +131,12 @@ _this is the recommended way to run remark42_
| image.resize-height | IMAGE_RESIZE_HEIGHT | `900` | height of resized image |
| auth.ttl.jwt | AUTH_TTL_JWT | `5m` | jwt TTL |
| auth.ttl.cookie | AUTH_TTL_COOKIE | `200h` | cookie TTL |
| auth.send-jwt-header | AUTH_SEND_JWT_HEADER | `false` | send JWT as a header instead of cookie |
| auth.same-site | AUTH_SAME_SITE | `default` | set same site policy for cookies (`default`, `none`, `lax` or `strict`)|
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
| auth.microsoft.cid | AUTH_MICROSOFT_CID | | Microsoft OAuth client ID |
| auth.microsoft.csec | AUTH_MICROSOFT_CSEC | | Microsoft OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | GitHub OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | GitHub OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
| auth.twitter.cid | AUTH_TWITTER_CID | | Twitter Consumer API Key |
| auth.twitter.csec | AUTH_TWITTER_CSEC | | Twitter Consumer API Secret key |
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
@@ -184,15 +177,12 @@ _this is the recommended way to run remark42_
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
| positive-score | POSITIVE_SCORE | `false` | restricts comment's score to be only positive |
| restricted-words | RESTRICTED_WORDS | | words banned in comments (can use `*`), _multi_ |
| restricted-names | RESTRICTED_NAMES | | names prohibited to use by the user, _multi_ |
| edit-time | EDIT_TIME | `5m` | edit window |
| read-age | READONLY_AGE | | read-only age of comments, days |
| image-proxy.http2https | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images |
| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false` | enable caching external images to current image storage |
| emoji | EMOJI | `false` | enable emoji support |
| simple-view | SIMPLE_VIEW | `false` | minimized UI with basic info only |
| proxy-cors | PROXY_CORS | `false` | disable internal CORS and delegate it to proxy |
| allowed-hosts | ALLOWED_HOSTS | enable all | limit hosts/sources allowed to embed comments |
| port | REMARK_PORT | `8080` | web server port |
| web-root | REMARK_WEB_ROOT | `./web` | web server root directory |
| update-limit | UPDATE_LIMIT | `0.5` | updates/sec limit |
@@ -294,14 +284,6 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
1. Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
1. Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
#### Microsoft Auth Provider
1. Register a new application [using the Azure portal](https://docs.microsoft.com/en-us/graph/auth-register-app-v2).
2. Under **"Authentication/Platform configurations/Web"** enter the correct url constructed as domain + `/auth/microsoft/callback`. i.e. `https://example.mysite.com/auth/microsoft/callback`
3. In "Overview" take note of the **Application (client) ID**
4. Choose the new project from the top right project dropdown (only if another project is selected)
5. Select "Certificates & secrets" and click on "+ New Client Secret".
##### Twitter Auth Provider
1. Create a new twitter application https://developer.twitter.com/en/apps
@@ -330,16 +312,11 @@ Optionally, anonymous access can be turned on. In this case an extra `anonymous`
- name should be at least 3 characters long
- name has to start from the letter and contains letters, numbers, underscores and spaces only.
### Importing comments
Remark supports importing comments from Disqus, WordPress or native backup format.
All imported comments has `Imported` field set to `true`.
#### Initial import from Disqus
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
3. Run import command - `docker exec -it remark42 import -p disqus -f /srv/var/{disqus-export-name}.xml -s {your site id}`
3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
#### Initial import from WordPress
@@ -459,14 +436,18 @@ Add this snippet to the bottom of web page:
max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
locale: 'en', // set up locale and language, if it isn't defined default value ('en') will be used
show_email_subscription: false // optional param; by default it is `true` and you can see email subscription feature
// in interface when enable it from backend side
// if you set this param in `false` you will get notifications email notifications as admin
// but your users won't have interface for subscription
locale: 'en' // set up locale and language, if it isn't defined default value ('en') will be used
};
(function(c) {
for(var i = 0; i < c.length; i++){
var d = document, s = d.createElement('script');
s.src = remark_config.host + '/web/' +c[i] +'.js';
s.defer = true;
(d.head || d.body).appendChild(s);
}
})(remark_config.components || ['embed']);
</script>
<script>!function(e,n){for(var o=0;o<e.length;o++){var r=n.createElement("script"),c=".js",d=n.head||n.body;"noModule"in r?(r.type="module",c=".mjs"):r.async=!0,r.defer=!0,r.src=remark_config.host+"/web/"+e[o]+c,d.appendChild(r)}}(remark_config.components||["embed"],document);</script>
```
And then add this node in the place where you want to see Remark42 widget:
@@ -477,8 +458,6 @@ And then add this node in the place where you want to see Remark42 widget:
After that widget will be rendered inside this node.
If you want to set this up on a Single Page App, see [appropriate doc page](https://remark42.com/docs/latest/spa/).
##### Themes
Right now Remark has two themes: light and dark.
@@ -512,6 +491,15 @@ Add this snippet to the bottom of web page, or adjust already present `remark_co
site_id: 'YOUR_SITE_ID',
components: ['last-comments']
};
(function(c) {
for(var i = 0; i < c.length; i++){
var d = document, s = d.createElement('script');
s.src = remark_config.host + '/web/' +c[i] +'.js';
s.defer = true;
(d.head || d.body).appendChild(s);
}
})(remark_config.components || ['embed']);
</script>
```
@@ -536,6 +524,15 @@ Add this snippet to the bottom of web page, or adjust already present `remark_co
site_id: 'YOUR_SITE_ID',
components: ['counter']
};
(function(c) {
for(var i = 0; i < c.length; i++){
var d = document, s = d.createElement('script');
s.src = remark_config.host + '/web/' +c[i] +'.js';
s.defer = true;
(d.head || d.body).appendChild(s);
}
})(remark_config.components || ['embed']);
</script>
```
@@ -565,14 +562,11 @@ To bring it up run:
```bash
# if you mainly work on backend
cp compose-dev-backend.yml compose-private.yml
docker-compose -f compose-dev-backend.yml build
docker-compose -f compose-dev-backend.yml up
# if you mainly work on frontend
cp compose-dev-frontend.yml compose-private.yml
# now, edit / debug `compose-private.yml` to your heart's content.
# build and run
docker-compose -f compose-private.yml build
docker-compose -f compose-private.yml up
docker-compose -f compose-dev-frontend.yml build
docker-compose -f compose-dev-frontend.yml up
```
It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
@@ -595,7 +589,7 @@ It stars backend service with embedded bolt store on port `8080` with basic auth
#### Developer guide
Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
Frontend guide can be found here: [./frontend/Readme.md](./frontend/Readme.md)
#### Build
You should have at least 2GB RAM or swap enabled for building
@@ -622,10 +616,7 @@ You can attach to locally running backend by providing `REMARK_URL` environment
npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
```
**Note** If you want to redefine env variables such as `PORT` on your local instance you can add `.env` file
to `./frontend` folder and rewrite variables as you wish. For such functional we use `dotenv`
The best way for start local developer environment:
The best way for start local developer enviroment:
```sh
cp compose-dev-frontend.yml compose-private-frontend.yml
docker-compose -f compose-private-frontend.yml up --build
-18
View File
@@ -1,18 +0,0 @@
# Security Policy
## Supported Versions
We release patches for security vulnerabilities.
| Version | Supported |
| ------- | ------------------ |
| current | :white_check_mark:
| 1.6.x | :white_check_mark: |
| <1.5.x | :x: |
## Reporting a Vulnerability
Please report (suspected) security vulnerabilities to umputun@gmail.com. You will receive a response from us within 48 hours.
If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.
+1 -8
View File
@@ -1,5 +1,4 @@
run:
timeout: 5m
output:
format: tab
skip-dirs:
@@ -26,11 +25,6 @@ linters-settings:
- experimental
disabled-checks:
- wrapperFunc
# TODO: feel free to remove these excludes and fix the code
- hugeParam
- rangeValCopy
- singleCaseSwitch
- ifElseChain
linters:
enable:
@@ -53,7 +47,6 @@ linters:
- stylecheck
- gochecknoinits
- scopelint
- gocritic
- nakedret
- gosimple
- prealloc
@@ -75,4 +68,4 @@ issues:
exclude-use-default: false
service:
golangci-lint-version: 1.31.x
golangci-lint-version: 1.23.x
+1 -1
View File
@@ -10,4 +10,4 @@ In order to run remark42 with memory_store copy provided `compose-dev-memstore.y
As usual, demo site will run on http://127.0.0.1:8080/web/
note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
In real-life usage `replace github.com/umputun/remark42/backend => ../../` should not be used.
In real-life usage `replace github.com/umputun/remark/backend => ../../` should not be used.
@@ -10,7 +10,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/admin"
)
// MemAdmin implements admin.Store with memory backend
@@ -35,7 +35,7 @@ func NewMemAdminStore(key string) *MemAdmin {
}
// Key executes find by siteID and returns substructure with secret key
func (m *MemAdmin) Key(_ string) (key string, err error) {
func (m *MemAdmin) Key() (key string, err error) {
return m.key, nil
}
@@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestMemAdmin_Get(t *testing.T) {
@@ -31,7 +31,7 @@ func TestMemAdmin_Get(t *testing.T) {
email, err := ms.Email("site1")
assert.NoError(t, err)
assert.Equal(t, "e1", email)
key, err := ms.Key("any")
key, err := ms.Key()
assert.NoError(t, err)
assert.Equal(t, "secret", key)
@@ -41,7 +41,7 @@ func TestMemAdmin_Get(t *testing.T) {
email, err = ms.Email("site2")
assert.NoError(t, err)
assert.Equal(t, "e2", email)
key, err = ms.Key("any")
key, err = ms.Key()
assert.NoError(t, err)
assert.Equal(t, "secret", key)
+13 -14
View File
@@ -14,8 +14,8 @@ import (
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
)
const lastLimit = 1000
@@ -535,19 +535,18 @@ func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error
func (m *MemData) updateComment(comment store.Comment) error {
comments := m.posts[comment.Locator.SiteID]
for i, c := range comments {
if c.ID != comment.ID || c.Locator != comment.Locator {
continue
if c.ID == comment.ID && c.Locator == comment.Locator {
c.Text = comment.Text
c.Orig = comment.Orig
c.Score = comment.Score
c.Votes = comment.Votes
c.Pin = comment.Pin
c.Deleted = comment.Deleted
c.User = comment.User
comments[i] = c
m.posts[comment.Locator.SiteID] = comments
return nil
}
c.Text = comment.Text
c.Orig = comment.Orig
c.Score = comment.Score
c.Votes = comment.Votes
c.Pin = comment.Pin
c.Deleted = comment.Deleted
c.User = comment.User
comments[i] = c
m.posts[comment.Locator.SiteID] = comments
return nil
}
return errors.New("not found")
}
@@ -15,8 +15,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
)
func TestMemData_CreateAndFind(t *testing.T) {
@@ -14,7 +14,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/image"
)
// MemImage implements image.Store with memory backend
+6 -6
View File
@@ -1,14 +1,14 @@
module github.com/umputun/remark42/memory_store
module github.com/umputun/remark/memory_store
go 1.14
require (
github.com/go-pkgz/jrpc v0.2.0
github.com/go-pkgz/lgr v0.10.4
github.com/go-pkgz/jrpc v0.1.0
github.com/go-pkgz/lgr v0.7.0
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.6.1
github.com/stretchr/testify v1.5.1
github.com/umputun/go-flags v1.5.1
github.com/umputun/remark42/backend v1.6.0
github.com/umputun/remark/backend v1.5.0
)
replace github.com/umputun/remark42/backend => ../../
replace github.com/umputun/remark/backend => ../../
+119 -146
View File
@@ -1,12 +1,16 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
github.com/alecthomas/chroma v0.6.0 h1:gcvXlpe0/NoQP3BvneRfgcauLIJDw9VblkoFwZ5XGFs=
github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
@@ -17,13 +21,13 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1p
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.11.4/go.mod h1:VL3UDEfAH59bSa7MuHMuFToxkqyHh69s/WUbYlOAuyg=
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -31,164 +35,121 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/didip/tollbooth/v6 v6.0.1 h1:QvLvRpB1G2bzKvkRze0muMUBlGN9H1z7tJ4DH4ypWOU=
github.com/didip/tollbooth/v6 v6.0.1/go.mod h1:j2pKs+JQ5PvU/K4jFnrnwntrmfUbYLJE5oSdxR37FD0=
github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9 h1:gTh8fKuI/yLqQtZEPlDX3ZGsiTPZIe0ADHsxXSbwO1I=
github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M=
github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/chi v4.1.0+incompatible h1:ETj3cggsVIY2Xao5ExCu6YhEh5MD6JTfcBzS37R260w=
github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
github.com/go-pkgz/auth v0.11.0/go.mod h1:NzVqlTW0E9JXVdAaWRq81XZjICgHnNaNdUfE3CbS2T4=
github.com/go-pkgz/auth v1.14.0/go.mod h1:1GVd61pXZcuJ0ZnOUdCTY08V8SreO7MJtsvEd5/WEWA=
github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
github.com/go-pkgz/jrpc v0.2.0 h1:CLy/eZyekjraVrxZV18N2R1mYLMJ/nWrgdfyIOGPY/E=
github.com/go-pkgz/jrpc v0.2.0/go.mod h1:wd8vtQ4CgtCnuqua6x2b1SKIgv0VSOh5Dn0uUITbiUE=
github.com/go-pkgz/lcw v0.7.1/go.mod h1:3P6g9QrJsDePXEMe42ywO+tW08L17tBJGwIDdI7lZ6g=
github.com/go-pkgz/lcw v0.8.1/go.mod h1:Xw0/ZfApATgbjVPYRZO4XHdWyxAjErDWDWJ7TLlw1Vc=
github.com/go-pkgz/auth v0.10.1/go.mod h1:wxyQqc0UUP1jT4l6zk1r6XPcVdcgIzW2OiQ8hBEHd64=
github.com/go-pkgz/jrpc v0.1.0 h1:hNg/IyfEqJcSWOKkuHw0ZwcuGc9TDp7QZREsD2ycmiM=
github.com/go-pkgz/jrpc v0.1.0/go.mod h1:JxZsvoBklA50DNhELVJnJ567Rt+KrMH9rR3u515wvE8=
github.com/go-pkgz/lcw v0.5.0/go.mod h1:CSdQRQthxJQ4iDD4wTPPuWFbFdknJzwJ8WXu1nfxb10=
github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/buntdb v1.0.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -197,100 +158,112 @@ github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt
github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/oauth2.v3 v3.12.0/go.mod h1:XEYgKqWX095YiPT+Aw5y3tCn+7/FMnlTFKrupgSiJ3I=
gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+2 -2
View File
@@ -14,8 +14,8 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/umputun/go-flags"
"github.com/umputun/remark42/memory_store/accessor"
"github.com/umputun/remark42/memory_store/server"
"github.com/umputun/remark/memory_store/accessor"
"github.com/umputun/remark/memory_store/server"
)
// opts with all cli commands and flags
@@ -11,17 +11,12 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/admin"
)
// get admin key
func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
var siteID string
if err := json.Unmarshal(params, &siteID); err != nil {
return jrpc.Response{Error: err.Error()}
}
key, err := s.adm.Key(siteID)
func (s *RPC) admKeyHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
key, err := s.adm.Key()
if err != nil {
return jrpc.Response{Error: err.Error()}
}
@@ -15,7 +15,7 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestRPC_admKeyHndl(t *testing.T) {
@@ -24,7 +24,7 @@ func TestRPC_admKeyHndl(t *testing.T) {
api := fmt.Sprintf("http://localhost:%d/test", port)
ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
key, err := ra.Key("any")
key, err := ra.Key()
assert.NoError(t, err)
assert.Equal(t, "secret", key)
}
+2 -2
View File
@@ -11,8 +11,8 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
)
func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
@@ -16,8 +16,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
)
func TestRPC_createHndl(t *testing.T) {
@@ -20,7 +20,7 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/image"
)
// gopher png for test, from https://golang.org/src/image/png/example_test.go
+3 -3
View File
@@ -9,9 +9,9 @@ package server
import (
"github.com/go-pkgz/jrpc"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
)
// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
@@ -17,7 +17,7 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/memory_store/accessor"
"github.com/umputun/remark/memory_store/accessor"
)
func chooseRandomUnusedPort() (port int) {
+2 -2
View File
@@ -72,12 +72,12 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
switch gr.Type {
case "fs":
if err := makeDirs(gr.FS.Path); err != nil {
return nil, errors.Wrap(err, "failed to create avatar store")
return nil, err
}
return avatar.NewLocalFS(gr.FS.Path), nil
case "bolt":
if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
return nil, errors.Wrap(err, "failed to create avatar store")
return nil, err
}
return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
}
+2 -2
View File
@@ -11,7 +11,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// CleanupCommand set of flags and command for cleanup
@@ -234,7 +234,7 @@ func (cc *CleanupCommand) setTitle(c store.Comment) error {
}
// isSpam calculates spam's probability as a score
func (cc *CleanupCommand) isSpam(comment store.Comment) (isSpam bool, spamScore float64) {
func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
badWord := func(txt string) float64 {
res := 0.0
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/umputun/go-flags"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
type cleanedComments struct {
+126 -183
View File
@@ -16,9 +16,9 @@ import (
"github.com/dgrijalva/jwt-go"
"github.com/go-pkgz/jrpc"
"github.com/go-pkgz/lcw/eventbus"
log "github.com/go-pkgz/lgr"
"github.com/kyokomi/emoji"
authcache "github.com/patrickmn/go-cache"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
@@ -29,16 +29,15 @@ import (
"github.com/go-pkgz/auth/token"
cache "github.com/go-pkgz/lcw"
"github.com/umputun/remark42/backend/app/migrator"
"github.com/umputun/remark42/backend/app/notify"
"github.com/umputun/remark42/backend/app/rest/api"
"github.com/umputun/remark42/backend/app/rest/proxy"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark42/backend/app/templates"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/notify"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
// ServerCommand with command line flags and env
@@ -51,6 +50,7 @@ type ServerCommand struct {
SMTP SMTPGroup `group:"smtp" namespace:"smtp" env-namespace:"SMTP"`
Image ImageGroup `group:"image" namespace:"image" env-namespace:"IMAGE"`
SSL SSLGroup `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
Stream StreamGroup `group:"stream" namespace:"stream" env-namespace:"STREAM"`
ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -72,25 +72,17 @@ type ServerCommand struct {
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
UpdateLimit float64 `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
RestrictedWords []string `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
RestrictedNames []string `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
EnableEmoji bool `long:"emoji" env:"EMOJI" description:"enable emoji"`
SimpleView bool `long:"simpler-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
ProxyCORS bool `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
AllowedHosts []string `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments"`
Auth struct {
TTL struct {
JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
SendJWTHeader bool `long:"send-jwt-header" env:"SEND_JWT_HEADER" description:"send JWT as a header instead of cookie"`
SameSite string `long:"same-site" env:"SAME_SITE" description:"set same site policy for cookies" choice:"default" choice:"none" choice:"lax" choice:"strict" default:"default"` // nolint
Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
Microsoft AuthGroup `group:"microsoft" namespace:"microsoft" env-namespace:"MICROSOFT" description:"Microsoft OAuth"`
Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
Twitter AuthGroup `group:"twitter" namespace:"twitter" env-namespace:"TWITTER" description:"Twitter OAuth"`
Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
@@ -106,7 +98,7 @@ type ServerCommand struct {
SMTPUserName string `long:"user" env:"USER" description:"[deprecated, use --smtp.username] enable TLS"`
TLS bool `long:"tls" env:"TLS" description:"[deprecated, use --smtp.tls] SMTP TCP connection timeout"`
TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"[deprecated, use --smtp.timeout] SMTP TCP connection timeout"`
MsgTemplate string `long:"template" env:"TEMPLATE" description:"[deprecated, message template file]" default:"email_confirmation_login.html.tmpl"`
MsgTemplate string `long:"template" env:"TEMPLATE" description:"message template file"`
} `group:"email" namespace:"email" env-namespace:"EMAIL"`
} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
@@ -167,9 +159,8 @@ type AvatarGroup struct {
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"redis_pub_sub" choice:"mem" choice:"none" default:"mem"` // nolint
RedisAddr string `long:"redis_addr" env:"REDIS_ADDR" default:"127.0.0.1:6379" description:"address of redis cache, turn redis cache on for distributed cache"`
Max struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` // nolint
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
@@ -181,7 +172,7 @@ type AdminGroup struct {
Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"` //nolint
Shared struct {
Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
Email []string `long:"email" env:"EMAIL" description:"admin emails" env-delim:","`
Email string `long:"email" env:"EMAIL" default:"" description:"admin email"`
} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
}
@@ -207,7 +198,7 @@ type NotifyGroup struct {
API string `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
Email struct {
From string `long:"from_address" env:"FROM" description:"from email address"`
From string `long:"fromAddress" env:"FROM" description:"from email address"`
VerificationSubject string `long:"verification_subj" env:"VERIFICATION_SUBJ" description:"verification message subject"`
AdminNotifications bool `long:"notify_admin" env:"ADMIN" description:"notify admin on new comments via ADMIN_SHARED_EMAIL"`
} `group:"email" namespace:"email" env-namespace:"EMAIL"`
@@ -223,6 +214,13 @@ type SSLGroup struct {
ACMEEmail string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
}
// StreamGroup define options for streaming apis
type StreamGroup struct {
RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
MaxActive int `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
}
// RPCGroup defines options for remote modules (plugins)
type RPCGroup struct {
API string `long:"api" env:"API" description:"rpc extension api url"`
@@ -235,7 +233,6 @@ type RPCGroup struct {
type LoadingCache interface {
Get(key cache.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
Flush(req cache.FlusherRequest) // evict matched records
Close() error
}
// serverApp holds all active objects
@@ -251,8 +248,6 @@ type serverApp struct {
imageService *image.Service
authenticator *auth.Service
terminated chan struct{}
authRefreshCache *authRefreshCache // stored only to close it properly on shutdown
}
// Execute is the entry point for "server" command, called by flag parser
@@ -309,9 +304,6 @@ func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
s.SMTP.TimeOut = s.Auth.Email.TimeOut
result = append(result, DeprecatedFlag{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"})
}
if s.Auth.Email.MsgTemplate != "email_confirmation_login.html.tmpl" {
result = append(result, DeprecatedFlag{Old: "auth.email.template", RemoveVersion: "1.9.0"})
}
if s.LegacyImageProxy && !s.ImageProxy.HTTP2HTTPS {
s.ImageProxy.HTTP2HTTPS = s.LegacyImageProxy
result = append(result, DeprecatedFlag{Old: "img-proxy", New: "image-proxy.http2https", RemoveVersion: "1.7.0"})
@@ -324,7 +316,7 @@ func (s *ServerCommand) HandleDeprecatedFlags() (result []DeprecatedFlag) {
func (s *ServerCommand) newServerApp() (*serverApp, error) {
if err := makeDirs(s.BackupLocation); err != nil {
return nil, errors.Wrap(err, "failed to create backup store")
return nil, err
}
if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
@@ -364,21 +356,14 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
loadingCache, err := s.makeCache()
if err != nil {
_ = dataService.Close()
return nil, errors.Wrap(err, "failed to make cache")
}
avatarStore, err := s.makeAvatarStore()
if err != nil {
_ = dataService.Close()
return nil, errors.Wrap(err, "failed to make avatar store")
}
authRefreshCache := newAuthRefreshCache()
authenticator, err := s.makeAuthenticator(dataService, avatarStore, adminStore, authRefreshCache)
if err != nil {
_ = dataService.Close()
return nil, errors.Wrap(err, "failed to make authenticator")
}
authenticator := s.makeAuthenticator(dataService, avatarStore, adminStore)
exporter := &migrator.Native{DataStore: dataService}
@@ -423,33 +408,39 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
sslConfig, err := s.makeSSLConfig()
if err != nil {
_ = dataService.Close()
return nil, errors.Wrap(err, "failed to make config of ssl server params")
}
srv := &api.Rest{
Version: s.Revision,
DataService: dataService,
WebRoot: s.WebRoot,
RemarkURL: s.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
Migrator: migr,
ReadOnlyAge: s.ReadOnlyAge,
SharedSecret: s.SharedSecret,
Authenticator: authenticator,
Cache: loadingCache,
NotifyService: notifyService,
SSLConfig: sslConfig,
UpdateLimiter: s.UpdateLimit,
ImageService: imageService,
Version: s.Revision,
DataService: dataService,
WebRoot: s.WebRoot,
RemarkURL: s.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
Migrator: migr,
ReadOnlyAge: s.ReadOnlyAge,
SharedSecret: s.SharedSecret,
Authenticator: authenticator,
Cache: loadingCache,
NotifyService: notifyService,
SSLConfig: sslConfig,
UpdateLimiter: s.UpdateLimit,
ImageService: imageService,
Streamer: &api.Streamer{
TimeOut: s.Stream.TimeOut,
Refresh: s.Stream.RefreshInterval,
MaxActive: int32(s.Stream.MaxActive),
},
EmailNotifications: emailNotifications,
EmojiEnabled: s.EnableEmoji,
AnonVote: s.AnonymousVote && s.RestrictVoteIP,
SimpleView: s.SimpleView,
ProxyCORS: s.ProxyCORS,
AllowedAncestors: s.AllowedHosts,
SendJWTHeader: s.Auth.SendJWTHeader,
}
// enable admin notifications only if admin email is set
if s.Notify.Email.AdminNotifications && s.Admin.Shared.Email != "" {
srv.AdminEmail = s.Admin.Shared.Email
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -458,25 +449,23 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
if s.Auth.Dev {
da, errDevAuth := authenticator.DevAuth()
if errDevAuth != nil {
_ = dataService.Close()
return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
}
devAuth = da
}
return &serverApp{
ServerCommand: s,
restSrv: srv,
migratorSrv: migr,
exporter: exporter,
devAuth: devAuth,
dataService: dataService,
avatarStore: avatarStore,
notifyService: notifyService,
imageService: imageService,
authenticator: authenticator,
terminated: make(chan struct{}),
authRefreshCache: authRefreshCache,
ServerCommand: s,
restSrv: srv,
migratorSrv: migr,
exporter: exporter,
devAuth: devAuth,
dataService: dataService,
avatarStore: avatarStore,
notifyService: notifyService,
imageService: imageService,
authenticator: authenticator,
terminated: make(chan struct{}),
}, nil
}
@@ -495,7 +484,7 @@ func (a *serverApp) run(ctx context.Context) error {
a.activateBackup(ctx) // runs in goroutine for each site
if a.Auth.Dev {
go a.devAuth.Run(ctx) // dev oauth2 server on :8084
go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
}
// staging images resubmit after restart of the app
@@ -517,12 +506,6 @@ func (a *serverApp) run(ctx context.Context) error {
if e := a.avatarStore.Close(); e != nil {
log.Printf("[WARN] failed to close avatar store, %s", e)
}
if e := a.restSrv.Cache.Close(); e != nil {
log.Printf("[WARN] failed to close rest server cache, %s", e)
}
if e := a.authRefreshCache.Close(); e != nil {
log.Printf("[WARN] failed to close auth authRefreshCache, %s", e)
}
a.notifyService.Close()
// call potentially infinite loop with cancellation after a minute as a safeguard
minuteCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
@@ -586,12 +569,12 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
switch s.Avatar.Type {
case "fs":
if err := makeDirs(s.Avatar.FS.Path); err != nil {
return nil, errors.Wrap(err, "failed to create avatar store")
return nil, err
}
return avatar.NewLocalFS(s.Avatar.FS.Path), nil
case "bolt":
if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
return nil, errors.Wrap(err, "failed to create avatar store")
return nil, err
}
return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
case "uri":
@@ -618,7 +601,7 @@ func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
return image.NewService(boltImageStore, imageServiceParams), nil
case "fs":
if err := makeDirs(s.Image.FS.Path); err != nil {
return nil, errors.Wrap(err, "failed to create pictures store")
return nil, err
}
return image.NewService(&image.FileSystem{
Location: s.Image.FS.Path,
@@ -642,15 +625,12 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
switch s.Admin.Type {
case "shared":
sharedAdminEmail := ""
if len(s.Admin.Shared.Email) == 0 { // no admin email, use admin@domain
if s.Admin.Shared.Email == "" { // no admin email, use admin@domain
if u, err := url.Parse(s.RemarkURL); err == nil {
sharedAdminEmail = "admin@" + u.Host
s.Admin.Shared.Email = "admin@" + u.Host
}
} else {
sharedAdminEmail = s.Admin.Shared.Email[0]
}
return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, sharedAdminEmail), nil
return admin.NewStaticStore(s.SharedSecret, s.Sites, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
case "rpc":
r := &admin.RPC{Client: jrpc.Client{
API: s.Admin.RPC.API,
@@ -667,17 +647,6 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
func (s *ServerCommand) makeCache() (LoadingCache, error) {
log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
switch s.Cache.Type {
case "redis_pub_sub":
redisPubSub, err := eventbus.NewRedisPubSub(s.Cache.RedisAddr, "remark42-cache")
if err != nil {
return nil, errors.Wrap(err, "cache backend initialization, redis PubSub initialisation")
}
backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items), cache.EventBus(redisPubSub))
if err != nil {
return nil, errors.Wrap(err, "cache backend initialization")
}
return cache.NewScache(backend), nil
case "mem":
backend, err := cache.NewLruCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
@@ -691,7 +660,29 @@ func (s *ServerCommand) makeCache() (LoadingCache, error) {
return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
}
func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
var msgTemplate = `
<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
</head>
<body>
<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and paste this text into token field on comments page</i></p>
<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
</div>
<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
</div>
</body>
</html>
`
func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
providers := 0
if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
@@ -706,10 +697,6 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
authenticator.AddProvider("facebook", s.Auth.Facebook.CID, s.Auth.Facebook.CSEC)
providers++
}
if s.Auth.Microsoft.CID != "" && s.Auth.Microsoft.CSEC != "" {
authenticator.AddProvider("microsoft", s.Auth.Microsoft.CID, s.Auth.Microsoft.CSEC)
providers++
}
if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" {
authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
providers++
@@ -738,33 +725,18 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
ContentType: s.Auth.Email.ContentType,
}
sndr := sender.NewEmailClient(params, log.Default())
tmpl, err := s.loadEmailTemplate()
if err != nil {
return err
}
authenticator.AddVerifProvider("email", tmpl, sndr)
authenticator.AddVerifProvider("email", s.loadEmailTemplate(), sndr)
}
if s.Auth.Anonymous {
log.Print("[INFO] anonymous access enabled")
var isValidAnonName = regexp.MustCompile(`^[\p{L}\d_ ]+$`).MatchString
var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
// don't allow anon with space prefix or suffix
if strings.HasPrefix(user, " ") || strings.HasSuffix(user, " ") {
log.Printf("[WARN] name %q has space as a suffix or prefix", user)
return false, nil
}
user = strings.TrimSpace(user)
if len(user) < 3 {
log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
return false, nil
}
if len(user) > 64 {
log.Printf("[WARN] name %q is too long, should be up to 64 characters", user)
return false, nil
}
if !isValidAnonName(user) {
log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
@@ -777,29 +749,22 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error {
if providers == 0 {
log.Printf("[WARN] no auth providers defined")
}
return nil
}
// loadEmailTemplate trying to get template from statik
func (s *ServerCommand) loadEmailTemplate() (string, error) {
var file []byte
var err error
if s.Auth.Email.MsgTemplate == "email_confirmation_login.html.tmpl" {
fs := templates.NewFS()
file, err = fs.ReadFile(s.Auth.Email.MsgTemplate)
} else {
// deprecated loading from an external file, should be removed before v1.9.0
file, err = ioutil.ReadFile(s.Auth.Email.MsgTemplate)
log.Printf("[INFO] template %s will be read from disk", s.Auth.Email.MsgTemplate)
// loadEmailTemplate trying to get template from opts MsgTemplate and default to embedded
// if not defined or failed to load
func (s *ServerCommand) loadEmailTemplate() string {
tmpl := msgTemplate
if s.Auth.Email.MsgTemplate != "" {
log.Printf("[DEBUG] load email template from %s", s.Auth.Email.MsgTemplate)
b, err := ioutil.ReadFile(s.Auth.Email.MsgTemplate)
if err == nil {
tmpl = string(b)
} else {
log.Printf("[WARN] failed to load email template from %s, %v", s.Auth.Email.MsgTemplate, err)
}
}
if err != nil {
return "", errors.Wrapf(err, "failed to read file %s", s.Auth.Email.MsgTemplate)
}
return string(file), nil
return tmpl
}
func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *auth.Service) (*notify.Service, error) {
@@ -838,9 +803,6 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
return tkn, nil
},
}
if s.Notify.Email.AdminNotifications {
emailParams.AdminEmails = s.Admin.Shared.Email
}
smtpParams := notify.SMTPParams{
Host: s.SMTP.Host,
Port: s.SMTP.Port,
@@ -861,7 +823,7 @@ func (s *ServerCommand) makeNotify(dataStore *service.DataStore, authenticator *
}
}
if len(destinations) > 0 {
if len(destinations) != 0 {
log.Printf("[INFO] make notify, types=%s", s.Notify.Type)
notifyService = notify.NewService(dataStore, s.Notify.QueueSize, destinations...)
}
@@ -889,8 +851,8 @@ func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
config.ACMELocation = s.SSL.ACMELocation
if s.SSL.ACMEEmail != "" {
config.ACMEEmail = s.SSL.ACMEEmail
} else if s.Admin.Type == "shared" && len(s.Admin.Shared.Email) != 0 {
config.ACMEEmail = s.Admin.Shared.Email[0]
} else if s.Admin.Type == "shared" && s.Admin.Shared.Email != "" {
config.ACMEEmail = s.Admin.Shared.Email
} else if u, e := url.Parse(s.RemarkURL); e == nil {
config.ACMEEmail = "admin@" + u.Hostname()
}
@@ -898,17 +860,15 @@ func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
return config, err
}
func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Store, admns admin.Store, authRefreshCache *authRefreshCache) (*auth.Service, error) {
func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Store, admns admin.Store) *auth.Service {
authenticator := auth.NewService(auth.Opts{
URL: strings.TrimSuffix(s.RemarkURL, "/"),
Issuer: "remark42",
TokenDuration: s.Auth.TTL.JWT,
CookieDuration: s.Auth.TTL.Cookie,
SendJWTHeader: s.Auth.SendJWTHeader,
SameSiteCookie: s.parseSameSite(s.Auth.SameSite),
SecureCookies: strings.HasPrefix(s.RemarkURL, "https://"),
SecretReader: token.SecretFunc(func(aud string) (string, error) { // get secret per site
return admns.Key("")
return admns.Key()
}),
ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
if c.User == nil {
@@ -922,12 +882,15 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
log.Printf("[WARN] can't read email for %s, %v", c.User.ID, err)
}
// don't allow anonymous and email with admin's name
if strings.HasPrefix(c.User.ID, "anonymous_") || strings.HasPrefix(c.User.ID, "email_") {
for _, a := range s.RestrictedNames {
if strings.EqualFold(strings.TrimSpace(c.User.Name), a) {
// don't allow anonymous with admin's name
if strings.HasPrefix(c.User.ID, "anonymous_") {
admins, err := admns.Admins(c.Audience)
if err != nil {
log.Printf("[WARN] can't get admins for %s, %v", c.Audience, err)
}
for _, a := range admins {
if strings.EqualFold(c.User.Name, a) {
c.User.SetBoolAttr("blocked", true)
log.Printf("[INFO] blocked %+v, attempt to impersonate (restricted names)", c.User)
break
}
}
@@ -950,48 +913,28 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
AvatarResizeLimit: s.Avatar.RszLmt,
AvatarRoutePath: "/api/v1/avatar",
Logger: log.Default(),
RefreshCache: authRefreshCache,
RefreshCache: newAuthRefreshCache(),
UseGravatar: true,
})
if err := s.addAuthProviders(authenticator); err != nil {
return nil, err
}
return authenticator, nil
}
func (s *ServerCommand) parseSameSite(ss string) http.SameSite {
switch strings.ToLower(ss) {
case "default":
return http.SameSiteDefaultMode
case "none":
return http.SameSiteNoneMode
case "lax":
return http.SameSiteLaxMode
case "strict":
return http.SameSiteStrictMode
default:
return http.SameSiteDefaultMode
}
s.addAuthProviders(authenticator)
return authenticator
}
// authRefreshCache used by authenticator to minimize repeatable token refreshes
type authRefreshCache struct {
cache.LoadingCache
*authcache.Cache
}
func newAuthRefreshCache() *authRefreshCache {
expirableCache, _ := cache.NewExpirableCache(cache.TTL(5 * time.Minute))
return &authRefreshCache{LoadingCache: expirableCache}
return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
}
// Get implements cache getter with key converted to string
func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
return c.LoadingCache.Peek(key.(string))
return c.Cache.Get(key.(string))
}
// Set implements cache setter with key converted to string
func (c *authRefreshCache) Set(key, value interface{}) {
_, _ = c.LoadingCache.Get(key.(string), func() (interface{}, error) { return value, nil })
c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
}
+22 -94
View File
@@ -18,7 +18,6 @@ import (
"github.com/dgrijalva/jwt-go"
"github.com/go-pkgz/auth/token"
"github.com/umputun/go-flags"
"go.uber.org/goleak"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -81,10 +80,10 @@ func TestServerApp_DevMode(t *testing.T) {
// send ping
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, "pong", string(body))
cancel()
@@ -134,12 +133,6 @@ func TestServerApp_AnonMode(t *testing.T) {
defer resp.Body.Close()
assert.Equal(t, http.StatusCreated, resp.StatusCode)
// try to login with non-latin name
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=Раз_Два%20%20Три_34567&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
// try to login with bad name
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark", port))
require.NoError(t, err)
@@ -147,32 +140,13 @@ func TestServerApp_AnonMode(t *testing.T) {
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// try to login with short name
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%%20%%20&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// try to login with name what have space in prefix
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%%20somebody&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// try to login with name what have space in suffix
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=somebody%%20&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// try to login with long name
ln := strings.Repeat("x", 65)
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=%s&aud=remark", port, ln))
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// try to login with admin name
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umpUtun&aud=remark", port))
resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=umputun&aud=remark", port))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -196,16 +170,16 @@ func TestServerApp_AnonMode(t *testing.T) {
app.Wait()
}
func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (tkn string, claims token.Claims) {
func getAuthFromCookie(t *testing.T, app *serverApp, resp *http.Response) (token string, claims token.Claims) {
var err error
for _, c := range resp.Cookies() {
if c.Name == "JWT" {
tkn = c.Value
token = c.Value
claims, err = app.restSrv.Authenticator.TokenService().Parse(c.Value)
require.NoError(t, err)
}
}
return tkn, claims
return token, claims
}
func TestServerApp_WithSSL(t *testing.T) {
@@ -318,7 +292,7 @@ func TestServerApp_Failed(t *testing.T) {
_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
assert.NoError(t, err)
_, err = opts.newServerApp()
assert.EqualError(t, err, "failed to create backup store: can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
t.Log(err)
// invalid url
@@ -341,19 +315,6 @@ func TestServerApp_Failed(t *testing.T) {
_, err = opts.newServerApp()
assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah")
t.Log(err)
// wrong redis location
opts = ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
p = flags.NewParser(&opts, flags.Default)
_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--cache.type=redis_pub_sub", "--cache.redis_addr=wrong_address"})
assert.NoError(t, err)
_, err = opts.newServerApp()
assert.EqualError(t, err,
"failed to make cache: cache backend initialization, redis PubSub initialisation: "+
"problem subscribing to channel remark42-cache on address wrong_address: "+
"dial tcp: address wrong_address: missing port in address")
t.Log(err)
}
func TestServerApp_Shutdown(t *testing.T) {
@@ -411,7 +372,6 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
"--auth.email.user=test_user",
"--auth.email.passwd=test_password",
"--auth.email.timeout=15s",
"--auth.email.template=file.tmpl",
}
assert.Empty(t, s.SMTP.Host)
assert.Empty(t, s.SMTP.Port)
@@ -430,7 +390,6 @@ func TestServerApp_DeprecatedArgs(t *testing.T) {
{Old: "auth.email.user", New: "smtp.username", RemoveVersion: "1.7.0"},
{Old: "auth.email.passwd", New: "smtp.password", RemoveVersion: "1.7.0"},
{Old: "auth.email.timeout", New: "smtp.timeout", RemoveVersion: "1.7.0"},
{Old: "auth.email.template", RemoveVersion: "1.9.0"},
},
deprecatedFlags)
assert.Equal(t, "smtp.example.org", s.SMTP.Host)
@@ -521,7 +480,7 @@ func TestServerAuthHooks(t *testing.T) {
req.Header.Set("X-JWT", tk)
resp, err := client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
defer resp.Body.Close()
assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
// add comment with no-aud claim
@@ -531,15 +490,15 @@ func TestServerAuthHooks(t *testing.T) {
require.NoError(t, err)
t.Logf("no-aud claims: %s", tkNoAud)
req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/",
"site": "remark"}}`))
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-631/",
"site": "remark"}}`))
require.NoError(t, err)
req.Header.Set("X-JWT", tkNoAud)
resp, err = client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "user without aud claim rejected, \n"+tkNoAud+"\n"+string(body))
// block user dev as admin
@@ -549,10 +508,10 @@ func TestServerAuthHooks(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err = client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
t.Log(string(b))
// try add a comment with blocked user
@@ -562,51 +521,29 @@ func TestServerAuthHooks(t *testing.T) {
req.Header.Set("X-JWT", tk)
resp, err = client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.True(t, resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized,
"blocked user can't post, \n"+tk+"\n"+string(body))
cancel()
app.Wait()
client.CloseIdleConnections()
}
func TestServer_loadEmailTemplate(t *testing.T) {
cmd := ServerCommand{}
cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl"
r, err := cmd.loadEmailTemplate()
assert.NoError(t, err)
r := cmd.loadEmailTemplate()
assert.Equal(t, "The token is {{.Token}}", r)
cmd.Auth.Email.MsgTemplate = "badpath.tmpl"
r, err = cmd.loadEmailTemplate()
assert.EqualError(t, err, "failed to read file badpath.tmpl: open badpath.tmpl: no such file or directory")
assert.Equal(t, r, "")
}
cmd.Auth.Email.MsgTemplate = ""
r = cmd.loadEmailTemplate()
assert.Contains(t, r, "Remark42</h1>")
func TestServerCommand_parseSameSite(t *testing.T) {
tbl := []struct {
inp string
res http.SameSite
}{
{"", http.SameSiteDefaultMode},
{"default", http.SameSiteDefaultMode},
{"blah", http.SameSiteDefaultMode},
{"none", http.SameSiteNoneMode},
{"lax", http.SameSiteLaxMode},
{"strict", http.SameSiteStrictMode},
}
cmd := ServerCommand{}
for i, tt := range tbl {
tt := tt
t.Run(strconv.Itoa(i), func(t *testing.T) {
assert.Equal(t, tt.res, cmd.parseSameSite(tt.inp))
})
}
cmd.Auth.Email.MsgTemplate = "bad-file"
r = cmd.loadEmailTemplate()
assert.Contains(t, r, "Remark42</h1>")
}
func chooseRandomUnusedPort() (port int) {
@@ -672,16 +609,12 @@ func prepServerApp(t *testing.T, fn func(o ServerCommand) ServerCommand) (*serve
cmd.SMTP.TimeOut = time.Second
cmd.UpdateLimit = 10
cmd.Admin.Type = "shared"
cmd.Admin.Shared.Admins = []string{"id1", "id2"}
cmd.RestrictedNames = []string{"umputun", "bobuk"}
cmd.Admin.Shared.Admins = []string{"umputun", "bobuk"}
cmd = fn(cmd)
os.Remove(cmd.Store.Bolt.Path + "/remark.db")
return createAppFromCmd(t, cmd)
}
func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Context, context.CancelFunc) {
// create app
app, err := cmd.newServerApp()
require.NoError(t, err)
@@ -689,8 +622,3 @@ func createAppFromCmd(t *testing.T, cmd ServerCommand) (*serverApp, context.Cont
rand.Seed(time.Now().UnixNano())
return app, ctx, cancel
}
func TestMain(m *testing.M) {
// ignore is added only for GitHub Actions, can't reproduce locally
goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"))
}
+2 -2
View File
@@ -10,7 +10,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/umputun/go-flags"
"github.com/umputun/remark42/backend/app/cmd"
"github.com/umputun/remark/backend/app/cmd"
)
// Opts with all cli commands and flags
@@ -84,7 +84,7 @@ func getDump() string {
return string(stacktrace[:length])
}
// nolint:gochecknoinits // can't avoid it in this place
// nolint:gochecknoinits
func init() {
// catch SIGQUIT and print stack traces
sigChan := make(chan os.Signal)
-10
View File
@@ -15,7 +15,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
func Test_Main(t *testing.T) {
@@ -87,12 +86,3 @@ func waitForHTTPServerStart(port int) {
}
}
}
func TestMain(m *testing.M) {
// both ignores are for leaks which are detected locally
goleak.VerifyTestMain(
m,
goleak.IgnoreTopFunction("github.com/umputun/remark42/backend/app.init.0.func1"),
goleak.IgnoreTopFunction("net/http.(*Server).Shutdown"),
)
}
+6 -6
View File
@@ -9,7 +9,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// Disqus implements Importer from disqus xml
@@ -52,8 +52,9 @@ type uid struct {
// Import from disqus and save to store
func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
if e := d.DataStore.DeleteAll(siteID); e != nil {
return 0, e
if err = d.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
commentsCh := d.convert(r, siteID)
@@ -135,10 +136,9 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
Text: d.cleanText(comment.Message),
Timestamp: comment.CreatedAt,
ParentID: comment.Pid.Val,
Imported: true,
}
if comment.AuthorUserName == "" { // empty comment.AuthorUserName from disqus
c.User.ID = "disqus_" + store.EncodeID(c.User.Name)
if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
c.User.ID = "disqus_" + c.User.Name
}
if c.ID == "" { // no comment.UID
c.ID = comment.ID
+7 -12
View File
@@ -10,10 +10,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestDisqus_Import(t *testing.T) {
@@ -39,11 +39,6 @@ func TestDisqus_Import(t *testing.T) {
assert.Equal(t, "Alexander Blah", c.User.Name)
assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID)
assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
assert.True(t, c.Imported)
c = last[1] // get comment with empty username
assert.Equal(t, "No Username", c.User.Name)
assert.Equal(t, "disqus_62e24ea213756cda0339e1074819f15e25214361", c.User.ID)
posts, err := dataStore.List("test", 0, 0)
assert.NoError(t, err)
@@ -76,7 +71,6 @@ func TestDisqus_Convert(t *testing.T) {
ID: "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434",
IP: "178.178.178.178",
},
Imported: true,
}
exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z")
assert.Equal(t, exp0, res[0])
@@ -175,9 +169,10 @@ var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>john.nousername@gmail.com</email>
<name>No Username</name>
<email>dmitri.noname@gmail.com</email>
<name>Dmitry Noname</name>
<isAnonymous>false</isAnonymous>
<username>google-74b9e7568ef6860e93862c5d77590123</username>
</author>
<ipAddress>89.89.89.139</ipAddress>
<thread dsq:id="247918464"/>
+2 -2
View File
@@ -10,8 +10,8 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
// Importer defines interface to convert posts from external sources
+4 -4
View File
@@ -10,10 +10,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_ImportDisqus(t *testing.T) {
+4 -5
View File
@@ -11,8 +11,8 @@ import (
"github.com/go-pkgz/syncs"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
const nativeVersion = 1
@@ -140,8 +140,8 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
return 0, errors.Errorf("unexpected import file version %d", m.Version)
}
if e := n.DataStore.DeleteAll(siteID); e != nil {
return 0, e
if err = n.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
var failed, total, comments int64
@@ -155,7 +155,6 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
for {
comment := store.Comment{}
err = dec.Decode(&comment)
comment.Imported = true
if err == io.EOF {
break
}
+9 -11
View File
@@ -14,10 +14,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestNative_Export(t *testing.T) {
@@ -73,7 +73,7 @@ func TestNative_Import(t *testing.T) {
inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00","imported":false}`
{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
b.AdminStore = admin.NewStaticStore("12345", nil, []string{}, "")
r := Native{DataStore: b}
@@ -87,12 +87,10 @@ func TestNative_Import(t *testing.T) {
assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
assert.Equal(t, "1234", comments[0].ParentID)
assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
assert.True(t, comments[0].Imported)
assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
assert.True(t, comments[1].Imported)
assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
@@ -180,11 +178,11 @@ func TestNative_ImportManyWithError(t *testing.T) {
}
// makes new boltdb, put two records
func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
func prep(t *testing.T) (*service.DataStore, func()) {
testDB := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
testDb := fmt.Sprintf("/tmp/migrator-%d.db", rand.Intn(999999999))
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDB})
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.NoError(t, err)
b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
@@ -209,6 +207,6 @@ func prep(t *testing.T) (ds *service.DataStore, teardown func()) {
return b, func() {
require.NoError(t, b.Close())
_ = os.Remove(testDB)
_ = os.Remove(testDb)
}
}
+3 -4
View File
@@ -9,7 +9,7 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
const wpTimeLayout = "2006-01-02 15:04:05"
@@ -61,8 +61,8 @@ func (w *WordPress) Convert(text string) string {
// Import comments from WP and save to store
func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
if e := w.DataStore.DeleteAll(siteID); e != nil {
return 0, e
if err = w.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
commentsCh := w.convert(r, siteID)
@@ -139,7 +139,6 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
Text: comment.Content,
Timestamp: comment.Date.time,
ParentID: comment.PID,
Imported: true,
}
commentsCh <- commentFormatter.Format(c)
stats.inpComments++
+4 -6
View File
@@ -10,10 +10,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestWordPress_Import(t *testing.T) {
@@ -42,7 +42,6 @@ func TestWordPress_Import(t *testing.T) {
ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
assert.Equal(t, ts, c.Timestamp)
assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
assert.True(t, c.Imported)
posts, err := dataStore.List(siteID, 0, 0)
assert.NoError(t, err)
@@ -78,7 +77,6 @@ func TestWordPress_Convert(t *testing.T) {
ID: "wordpress_" + store.EncodeID("Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual"),
IP: "74.200.244.101",
},
Imported: true,
}
exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
assert.Equal(t, exp1, comments[1])
+150 -105
View File
@@ -6,7 +6,6 @@ import (
"crypto/tls"
"fmt"
"io"
"mime"
"mime/quotedprintable"
"net"
"net/smtp"
@@ -15,21 +14,17 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/go-pkgz/repeater"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/templates"
)
// EmailParams contain settings for email notifications
type EmailParams struct {
From string // from email address
AdminEmails []string // administrator emails to send copy of comment notification to
MsgTemplatePath string // path to request message template
VerificationSubject string // verification message sub
VerificationTemplatePath string // path to verification template
SubscribeURL string // full subscribe handler URL
UnsubscribeURL string // full unsubscribe handler URL
From string // from email address
MsgTemplate string // request message template
VerificationSubject string // verification message subject
VerificationTemplate string // verification message template
SubscribeURL string // full subscribe handler URL
UnsubscribeURL string // full unsubscribe handler URL
TokenGenFn func(userID, email, site string) (string, error) // Unsubscribe token generation function
}
@@ -106,123 +101,174 @@ type verifyTmplData struct {
}
const (
defaultVerificationSubject = "Email verification"
defaultEmailTimeout = 10 * time.Second
defaultEmailTemplatePath = "email_reply.html.tmpl"
defaultEmailVerificationTemplatePath = "email_confirmation_subscription.html.tmpl"
defaultVerificationSubject = "Email verification"
defaultEmailTimeout = 10 * time.Second
defaultEmailTemplate = `<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<style type="text/css">
img {
max-width: 100%;
max-height: 250px;
margin: 5px 0;
display: block;
color: #000;
}
a {
text-decoration: none;
color: #0aa;
}
p {
margin: 0 0 12px;
}
blockquote {
margin: 10px 0;
padding: 12px 12px 1px 12px;
background: rgba(255,255,255,.5)
}
</style>
</head>
<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
<body>
<div style="font-family: Helvetica, Arial, sans-serif; font-size: 18px; width: 100%; max-width: 640px; margin: auto;">
<h1 style="text-align: center; position: relative; color: #4fbbd6; margin-top: 10px; margin-bottom: 10px;">Remark42</h1>
{{- if .ForAdmin}}
<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
{{- else }}
<div style="font-size: 16px; text-align: center; margin-bottom: 10px; color:#000!important;">New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}</div>
{{- end }}
<div style="background-color: #eee; padding: 15px 20px 20px 20px; border-radius: 3px;">
{{- if .ParentCommentText}}
<div style="margin-bottom: 12px; line-height: 24px; word-break: break-all;">
<img src="{{.ParentUserPicture}}" style="width: 24px; height: 24px; display: inline; vertical-align: middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
<span style="font-size: 14px; font-weight: bold; color: #777">{{.ParentUserName}}</span>
<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}</span>
<a href="{{.ParentCommentLink}}" style="color: #0aa; font-size: 14px;"><b>Show</b></a>
</div>
<div style="font-size: 14px; color:#333!important; padding: 0 14px 0 2px; border-radius: 3px; line-height: 1.4;">
{{.ParentCommentText}}
</div>
{{- end }}
<div style="padding-left: 20px; border-left: 1px dotted rgba(0,0,0,0.15); margin-top: 15px; padding-top: 5px;">
<div style="margin-bottom: 12px;" line-height: 24px;word-break: break-all;>
<img src="{{.UserPicture}}" style="width: 24px; height: 24px; display:inline; vertical-align:middle; margin: 0 8px 0 0; border-radius: 3px; background-color: #ccc;"/>
<span style="font-size: 14px; font-weight: bold; color: #777">{{.UserName}}</span>
<span style="color: #999; font-size: 14px; margin: 0 8px;">{{.CommentDate.Format "02.01.2006 at 15:04"}}</span>
<a href="{{.CommentLink}}" style="color: #0aa; font-size: 14px;"><b>Reply</b></a>
</div>
<div style="font-size: 16px; background-color: #fff; color:#000!important; padding: 14px 14px 2px 14px; border-radius: 3px; line-height: 1.4;">{{.CommentText}}</div>
</div>
</div>
<div style="text-align: center; font-size: 14px; margin-top: 32px;">
<i style="color: #000!important;">Sent to <a style="color:inherit; text-decoration: none" href="mailto:{{.Email}}">{{.Email}}</a>{{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}</i>
<div style="margin: auto; width: 150px; border-top: 1px solid rgba(0, 0, 0, 0.15); padding-top: 15px; margin-top: 15px;"></div>
{{- if .UnsubscribeLink}}
<a style="color: #0aa;" href="{{.UnsubscribeLink}}">Unsubscribe</a>
{{- end }}
<!-- This is hack for remove collapser in Gmail which can collapse end of the message -->
<div style="opacity: 0;font-size: 1;">[{{.CommentDate.Format "02.01.2006 at 15:04"}}]</div>
</div>
</div>
</body>
</html>
`
defaultEmailVerificationTemplate = `<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
</head>
<body>
<!-- Some of blocks on this page have color: #000 because GMail can wrap block in his own tags which can change text color -->
<div style="text-align: center; font-family: Helvetica, Arial, sans-serif; font-size: 18px;">
<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em; color:#000!important;">Confirmation for <b>{{.User}}</b> on site <b>{{.Site}}</b></p>
{{- if .SubscribeURL}}
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;"><a href="{{.SubscribeURL}}{{.Token}}">Click here to subscribe to email notifications</a></p>
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">Alternatively, you can use code below for subscription.</p>
{{- end }}
<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
<p style="position: relative; margin: 0 0 0.5em 0;color:#000!important;">TOKEN</p>
<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i style="color:#000!important;">Copy and paste this text into token field on comments page</i></p>
<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
</div>
<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i style="color:#000!important;">Sent to {{.Email}}</i></p>
</div>
</body>
</html>
`
)
// NewEmail makes new Email object, returns error in case of e.MsgTemplate or e.VerificationTemplate parsing error
func NewEmail(emailParams EmailParams, smtpParams SMTPParams) (*Email, error) {
// set up Email emailParams
res := Email{EmailParams: emailParams}
if res.MsgTemplate == "" {
res.MsgTemplate = defaultEmailTemplate
}
if res.VerificationTemplate == "" {
res.VerificationTemplate = defaultEmailVerificationTemplate
}
if res.VerificationSubject == "" {
res.VerificationSubject = defaultVerificationSubject
}
// set up SMTP emailParams
res.smtp = &emailClient{}
res.SMTPParams = smtpParams
if res.TimeOut <= 0 {
res.TimeOut = defaultEmailTimeout
}
if res.VerificationSubject == "" {
res.VerificationSubject = defaultVerificationSubject
}
// initialize templates
err := res.setTemplates()
if err != nil {
return nil, errors.Wrap(err, "can't set templates")
}
log.Printf("[DEBUG] Create new email notifier for server %s with user %s, timeout=%s",
res.Host, res.Username, res.TimeOut)
return &res, nil
}
func (e *Email) setTemplates() error {
// initialize templates
var err error
var msgTmplFile, verifyTmplFile []byte
fs := templates.NewFS()
if e.VerificationTemplatePath == "" {
e.VerificationTemplatePath = defaultEmailVerificationTemplatePath
if res.msgTmpl, err = template.New("messageFromRequest").Parse(res.MsgTemplate); err != nil {
return nil, errors.Wrapf(err, "can't parse message template")
}
if e.MsgTemplatePath == "" {
e.MsgTemplatePath = defaultEmailTemplatePath
if res.verifyTmpl, err = template.New("messageFromRequest").Parse(res.VerificationTemplate); err != nil {
return nil, errors.Wrapf(err, "can't parse verification template")
}
if msgTmplFile, err = fs.ReadFile(e.MsgTemplatePath); err != nil {
return errors.Wrapf(err, "can't read message template")
}
if verifyTmplFile, err = fs.ReadFile(e.VerificationTemplatePath); err != nil {
return errors.Wrapf(err, "can't read verification template")
}
if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil {
return errors.Wrapf(err, "can't parse message template")
}
if e.verifyTmpl, err = template.New("verifyTmpl").Parse(string(verifyTmplFile)); err != nil {
return errors.Wrapf(err, "can't parse verification template")
}
return nil
return &res, err
}
// Send email about comment reply to Request.Emails and Email.AdminEmails
// if they're set.
// Send email about comment reply to Request.Email if it's set,
// also sends email to site administrator if appropriate option is set.
// Thread safe
func (e *Email) Send(ctx context.Context, req Request) error {
select {
case <-ctx.Done():
return errors.Errorf("sending email messages about comment %q aborted due to canceled context", req.Comment.ID)
default:
}
result := new(multierror.Error)
for _, email := range req.Emails {
err := e.buildAndSendMessage(ctx, req, email, false)
result = multierror.Append(errors.Wrapf(err, "problem sending user email notification to %q", email))
}
for _, email := range e.AdminEmails {
err := e.buildAndSendMessage(ctx, req, email, true)
result = multierror.Append(errors.Wrapf(err, "problem sending admin email notification to %q", email))
}
return result.ErrorOrNil()
}
func (e *Email) buildAndSendMessage(ctx context.Context, req Request, email string, forAdmin bool) error {
log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
msg, err := e.buildMessageFromRequest(req, email, forAdmin)
if err != nil {
return err
}
return repeater.NewDefault(5, time.Millisecond*250).Do(
ctx,
func() error {
return e.sendMessage(emailMessage{from: e.From, to: email, message: msg})
})
}
// SendVerification email verification VerificationRequest.Email if it's set.
// Thread safe
func (e *Email) SendVerification(ctx context.Context, req VerificationRequest) error {
func (e *Email) Send(ctx context.Context, req Request) (err error) {
if req.Email == "" {
// this means we can't send this request via Email
return nil
}
select {
case <-ctx.Done():
return errors.Errorf("sending message to %q aborted due to canceled context", req.User)
return errors.Errorf("sending message to %q aborted due to canceled context", req.Email)
default:
}
var msg string
log.Printf("[DEBUG] send verification via %s, user %s", e, req.User)
msg, err := e.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
if err != nil {
return err
if req.Verification.Token != "" {
log.Printf("[DEBUG] send verification via %s, user %s", e, req.Verification.User)
msg, err = e.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
if err != nil {
return err
}
}
if req.Comment.ID != "" {
if req.parent.User.ID == req.Comment.User.ID && !req.ForAdmin {
// don't send anything if if user replied to their own comment
return nil
}
log.Printf("[DEBUG] send notification via %s, comment id %s", e, req.Comment.ID)
msg, err = e.buildMessageFromRequest(req, req.ForAdmin)
if err != nil {
return err
}
}
return repeater.NewDefault(5, time.Millisecond*250).Do(
@@ -250,16 +296,16 @@ func (e *Email) buildVerificationMessage(user, email, token, site string) (strin
}
// buildMessageFromRequest generates email message based on Request using e.MsgTemplate
func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool) (string, error) {
func (e *Email) buildMessageFromRequest(req Request, forAdmin bool) (string, error) {
subject := "New reply to your comment"
if forAdmin {
subject = "New comment to your site"
}
if req.Comment.PostTitle != "" {
subject += fmt.Sprintf(" for %q", req.Comment.PostTitle)
subject += fmt.Sprintf(" for \"%s\"", req.Comment.PostTitle)
}
token, err := e.TokenGenFn(req.parent.User.ID, email, req.Comment.Locator.SiteID)
token, err := e.TokenGenFn(req.parent.User.ID, req.Email, req.Comment.Locator.SiteID)
if err != nil {
return "", errors.Wrapf(err, "error creating token for unsubscribe link")
}
@@ -277,7 +323,7 @@ func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool
CommentLink: commentURLPrefix + req.Comment.ID,
CommentDate: req.Comment.Timestamp,
PostTitle: req.Comment.PostTitle,
Email: email,
Email: req.Email,
UnsubscribeLink: unsubscribeLink,
ForAdmin: forAdmin,
}
@@ -293,7 +339,7 @@ func (e *Email) buildMessageFromRequest(req Request, email string, forAdmin bool
if err != nil {
return "", errors.Wrapf(err, "error executing template to build comment reply message")
}
return e.buildMessage(subject, msg.String(), email, "text/html", unsubscribeLink)
return e.buildMessage(subject, msg.String(), req.Email, "text/html", unsubscribeLink)
}
// buildMessage generates email message to send using net/smtp.Data()
@@ -304,7 +350,7 @@ func (e *Email) buildMessage(subject, body, to, contentType, unsubscribeLink str
}
message = addHeader(message, "From", e.From)
message = addHeader(message, "To", to)
message = addHeader(message, "Subject", mime.BEncoding.Encode("utf-8", subject))
message = addHeader(message, "Subject", subject)
message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable")
if contentType != "" {
@@ -405,7 +451,6 @@ func (s *emailClient) Create(params SMTPParams) (smtpClient, error) {
tlsConf := &tls.Config{
InsecureSkipVerify: false,
ServerName: params.Host,
MinVersion: tls.VersionTLS12,
}
conn, err := tls.Dial("tcp", srvAddress, tlsConf)
if err != nil {
+97 -166
View File
@@ -12,92 +12,84 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestEmailNew(t *testing.T) {
emailParams := EmailParams{
From: "test@from",
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
}
smtpParams := SMTPParams{
Host: "test@host",
Port: 1000,
TLS: true,
Username: "test@username",
Password: "test@password",
TimeOut: time.Second,
}
email, err := NewEmail(emailParams, smtpParams)
assert.NoError(t, err)
assert.NotNil(t, email, "email returned")
assert.NotNil(t, email.msgTmpl, "e.template is set")
assert.Equal(t, emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
if smtpParams.TimeOut == 0 {
assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
} else {
assert.Equal(t, smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
}
assert.Equal(t, smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
assert.Equal(t, smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
assert.Equal(t, smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
assert.Equal(t, smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
assert.Equal(t, smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
}
func Test_initTemplatesErr(t *testing.T) {
testSet := []struct {
var testSet = []struct {
name string
err bool
errText string
emailParams EmailParams
smtpParams SMTPParams
}{
{
name: "with wrong path to verification template",
errText: "can't read verification template: open notfount.tmpl: no such file or directory",
{name: "empty"},
{name: "with template parse error",
err: true, errText: "can't parse message template: template: messageFromRequest:1: unexpected unclosed action in command",
emailParams: EmailParams{
VerificationTemplatePath: "notfount.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
MsgTemplate: "{{",
}},
{name: "with verification template parse error",
err: true, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
emailParams: EmailParams{
From: "test@from",
VerificationTemplate: "{{",
},
smtpParams: SMTPParams{
Host: "test@host",
Port: 1000,
TLS: true,
Username: "test@username",
Password: "test@password",
TimeOut: time.Second,
},
},
{
name: "with wrong path to message template",
errText: "can't read message template: open notfount.tmpl: no such file or directory",
{name: "normal creation",
err: false, errText: "can't parse verification template: template: messageFromRequest:1: unexpected unclosed action in command",
emailParams: EmailParams{
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "notfount.tmpl",
From: "test@from",
},
},
{
name: "with error on read verification template",
errText: "can't parse verification template: template: verifyTmpl",
emailParams: EmailParams{
VerificationTemplatePath: "testdata/bad.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
},
},
{
name: "with error on read message template",
errText: "can't parse message template: template: msgTmpl",
emailParams: EmailParams{
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/bad.html.tmpl",
smtpParams: SMTPParams{
Host: "test@host",
Port: 1000,
TLS: true,
Username: "test@username",
Password: "test@password",
TimeOut: time.Second,
},
},
}
for _, d := range testSet {
d := d
t.Run(d.name, func(t *testing.T) {
e := Email{EmailParams: d.emailParams}
err := e.setTemplates()
require.Error(t, err)
assert.Contains(t, err.Error(), d.errText)
email, err := NewEmail(d.emailParams, d.smtpParams)
if d.err && d.errText == "" {
assert.Error(t, err)
assert.Nil(t, email)
} else if d.err && d.errText != "" {
assert.EqualError(t, err, d.errText)
assert.Nil(t, email)
} else {
assert.NoError(t, err)
assert.NotNil(t, email, "email returned")
assert.NotNil(t, email.msgTmpl, "e.template is set")
assert.Equal(t, defaultEmailTemplate, email.EmailParams.MsgTemplate, "empty emailParams.MsgTemplate changed to default")
assert.Equal(t, defaultEmailVerificationTemplate, email.EmailParams.VerificationTemplate, "empty emailParams.VerificationTemplate changed to default")
assert.Equal(t, d.emailParams.From, email.EmailParams.From, "emailParams.From unchanged after creation")
if d.smtpParams.TimeOut == 0 {
assert.Equal(t, defaultEmailTimeout, email.TimeOut, "empty emailParams.TimeOut changed to default")
} else {
assert.Equal(t, d.smtpParams.TimeOut, email.TimeOut, "emailParams.TimOut unchanged after creation")
}
assert.Equal(t, d.smtpParams.Host, email.Host, "emailParams.Host unchanged after creation")
assert.Equal(t, d.smtpParams.Username, email.Username, "emailParams.Username unchanged after creation")
assert.Equal(t, d.smtpParams.Password, email.Password, "emailParams.Password unchanged after creation")
assert.Equal(t, d.smtpParams.Port, email.Port, "emailParams.Port unchanged after creation")
assert.Equal(t, d.smtpParams.TLS, email.TLS, "emailParams.TLS unchanged after creation")
}
})
}
}
@@ -109,39 +101,41 @@ func TestEmailSendErrors(t *testing.T) {
e.verifyTmpl, err = template.New("test").Parse("{{.Test}}")
assert.NoError(t, err)
assert.EqualError(t, e.SendVerification(context.Background(), VerificationRequest{Email: "bad@example.org", Token: "some"}),
assert.EqualError(t, e.Send(context.Background(), Request{Email: "bad@example.org", Verification: VerificationMetadata{Token: "some"}}),
"error executing template to build verification message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.verifyTmplData")
e.verifyTmpl, err = template.New("test").Parse(defaultEmailVerificationTemplate)
assert.NoError(t, err)
e.msgTmpl, err = template.New("test").Parse("{{.Test}}")
assert.NoError(t, err)
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\": "+
"error executing template to build comment reply message: "+
"template: test:1:2: executing \"test\" at <.Test>: "+
"can't evaluate field Test in type notify.msgTmplData\n\n")
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
"error executing template to build comment reply message: template: test:1:2: executing \"test\" at <.Test>: can't evaluate field Test in type notify.msgTmplData")
e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
assert.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
cancel()
assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Emails: []string{"bad@example.org"}}),
"sending email messages about comment \"999\" aborted due to canceled context")
assert.EqualError(t, e.Send(ctx, Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "test"}}, Email: "bad@example.org"}),
"sending message to \"bad@example.org\" aborted due to canceled context")
e.smtp = &fakeTestSMTP{}
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Emails: []string{"bad@example.org"}}),
"1 error occurred:\n\t* problem sending user email notification to \"bad@example.org\":"+
" error creating token for unsubscribe link: token generation error\n\n")
assert.EqualError(t, e.Send(context.Background(), Request{Comment: store.Comment{ID: "999"}, parent: store.Comment{User: store.User{ID: "error"}}, Email: "bad@example.org"}),
"error creating token for unsubscribe link: token generation error")
e.msgTmpl, err = template.New("test").Parse(defaultEmailTemplate)
assert.NoError(t, err)
}
func TestEmailSend_ExitConditions(t *testing.T) {
email, err := NewEmail(EmailParams{
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
}, SMTPParams{})
email, err := NewEmail(EmailParams{}, SMTPParams{})
assert.NoError(t, err)
assert.NotNil(t, email, "expecting email returned")
// prevent triggering e.autoFlush creation
emptyRequest := Request{Comment: store.Comment{ID: "999"}}
assert.NoError(t, email.Send(context.Background(), emptyRequest),
"Message without Emails and AdminEmails is not sent and returns nil")
"Message without parent comment User.Email is not sent and returns nil")
requestWithEqualUsersWithEmails := Request{Comment: store.Comment{ID: "999"}, Email: "good_example@example.org"}
assert.NoError(t, email.Send(context.Background(), requestWithEqualUsersWithEmails),
"Message with parent comment User equals comment User is not sent and returns nil")
}
func TestEmailSendClientError(t *testing.T) {
@@ -186,21 +180,8 @@ func TestEmailSendClientError(t *testing.T) {
"e.send called without smtpClient set returns error")
}
func TestEmail_DefaultTemplates(t *testing.T) {
email, err := NewEmail(EmailParams{}, SMTPParams{})
assert.Error(t, err)
assert.Nil(t, email)
email, err = NewEmail(EmailParams{VerificationTemplatePath: "testdata/verification.html.tmpl"}, SMTPParams{})
assert.Error(t, err)
assert.Nil(t, email)
}
func TestEmail_Send(t *testing.T) {
email, err := NewEmail(EmailParams{
From: "from@example.org",
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
}, SMTPParams{})
email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
assert.NoError(t, err)
assert.NotNil(t, email)
fakeSMTP := fakeTestSMTP{}
@@ -210,14 +191,14 @@ func TestEmail_Send(t *testing.T) {
req := Request{
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "test_title"},
parent: store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
Emails: []string{"test@example.org"},
Email: "test@example.org",
}
assert.NoError(t, email.Send(context.TODO(), req))
assert.Equal(t, "from@example.org", fakeSMTP.readMail())
assert.Equal(t, 1, fakeSMTP.readQuitCount())
assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
// test buildMessageFromRequest separately for message text
res, err := email.buildMessageFromRequest(req, req.Emails[0], false)
res, err := email.buildMessageFromRequest(req, req.ForAdmin)
assert.NoError(t, err)
assert.Contains(t, res, `From: from@example.org
To: test@example.org
@@ -229,17 +210,14 @@ List-Unsubscribe-Post: List-Unsubscribe=One-Click
List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
Date: `)
// send email to both user and admin, without parent set
email.AdminEmails = []string{"admin@example.org"}
// send email to admin without parent set
req = Request{
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
Emails: []string{"test@example.org"},
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, PostTitle: "test_title"},
Email: "admin@example.org",
ForAdmin: true,
}
assert.NoError(t, email.Send(context.TODO(), req))
assert.Equal(t, "from@example.org", fakeSMTP.readMail())
assert.Equal(t, 3, fakeSMTP.readQuitCount(), "plus two emails: one for user and one for admin")
assert.Equal(t, "admin@example.org", fakeSMTP.readRcpt())
res, err = email.buildMessageFromRequest(req, email.AdminEmails[0], true)
res, err = email.buildMessageFromRequest(req, req.ForAdmin)
assert.NoError(t, err)
assert.Contains(t, res, `From: from@example.org
To: admin@example.org
@@ -250,74 +228,27 @@ Content-Type: text/html; charset="UTF-8"
Date: `)
}
func TestEmail_SendWithUnicodeInSubject(t *testing.T) {
email, err := NewEmail(EmailParams{
From: "from@example.org",
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
}, SMTPParams{})
assert.NoError(t, err)
assert.NotNil(t, email)
fakeSMTP := fakeTestSMTP{}
email.smtp = &fakeSMTP
email.TokenGenFn = TokenGenFn
email.UnsubscribeURL = "https://remark42.com/api/v1/email/unsubscribe"
req := Request{
Comment: store.Comment{ID: "999", User: store.User{ID: "1", Name: "test_user"}, ParentID: "1", PostTitle: "Привет"},
parent: store.Comment{ID: "1", User: store.User{ID: "999", Name: "parent_user"}},
Emails: []string{"test@example.org"},
}
// test buildMessageFromRequest separately for message text
res, err := email.buildMessageFromRequest(req, req.Emails[0], false)
assert.NoError(t, err)
// `=?utf-8?b?TmV3IHJlcGx5IHRvIHlvdXIgY29tbWVudCBmb3IgItCf0YDQuNCy0LXRgiI=?=` -> `New reply to your comment for "Привет"` in base64 + required prefix and suffix
assert.Contains(t, res, `From: from@example.org
To: test@example.org
Subject: =?utf-8?b?TmV3IHJlcGx5IHRvIHlvdXIgY29tbWVudCBmb3IgItCf0YDQuNCy0LXRgiI=?=
Content-Transfer-Encoding: quoted-printable
MIME-version: 1.0
Content-Type: text/html; charset="UTF-8"
List-Unsubscribe-Post: List-Unsubscribe=One-Click
List-Unsubscribe: <https://remark42.com/api/v1/email/unsubscribe?site=&tkn=token>
Date: `)
}
func TestEmail_SendVerification(t *testing.T) {
email, err := NewEmail(EmailParams{
From: "from@example.org",
VerificationTemplatePath: "testdata/verification.html.tmpl",
MsgTemplatePath: "testdata/msg.html.tmpl",
}, SMTPParams{})
email, err := NewEmail(EmailParams{From: "from@example.org"}, SMTPParams{})
assert.NoError(t, err)
assert.NotNil(t, email)
fakeSMTP := fakeTestSMTP{}
email.smtp = &fakeSMTP
email.TokenGenFn = TokenGenFn
// proper VerificationRequest without email
req := VerificationRequest{
SiteID: "remark",
User: "test_username",
Token: "secret_",
req := Request{
Email: "test@example.org",
Verification: VerificationMetadata{
SiteID: "remark",
User: "test_username",
Token: "secret_",
},
}
assert.NoError(t, email.SendVerification(context.TODO(), req))
assert.Equal(t, "", fakeSMTP.readMail())
assert.Equal(t, 0, fakeSMTP.readQuitCount())
assert.Equal(t, "", fakeSMTP.readRcpt())
// proper VerificationRequest with email
req.Email = "test@example.org"
assert.NoError(t, email.SendVerification(context.TODO(), req))
assert.NoError(t, email.Send(context.TODO(), req))
assert.Equal(t, "from@example.org", fakeSMTP.readMail())
assert.Equal(t, 1, fakeSMTP.readQuitCount())
assert.Equal(t, "test@example.org", fakeSMTP.readRcpt())
// VerificationRequest with canceled context
ctx, cancel := context.WithCancel(context.TODO())
cancel()
assert.EqualError(t, email.SendVerification(ctx, req), "sending message to \"test_username\" aborted due to canceled context")
// test buildVerificationMessage separately for message text
res, err := email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
// test buildMessageFromRequest separately for message text
res, err := email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
assert.NoError(t, err)
assert.Contains(t, res, `From: from@example.org
To: test@example.org
@@ -329,7 +260,7 @@ Date: `)
assert.Contains(t, res, `secret_`)
assert.NotContains(t, res, `https://example.org/`)
email.SubscribeURL = "https://example.org/subscribe.html?token="
res, err = email.buildVerificationMessage(req.User, req.Email, req.Token, req.SiteID)
res, err = email.buildVerificationMessage(req.Verification.User, req.Email, req.Verification.Token, req.Verification.SiteID)
assert.NoError(t, err)
assert.Contains(t, res, `From: from@example.org
To: test@example.org
+40 -106
View File
@@ -9,15 +9,14 @@ import (
log "github.com/go-pkgz/lgr"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// Service delivers notifications to multiple destinations
type Service struct {
dataService Store
destinations []Destination
queue chan Request
verificationQueue chan VerificationRequest
dataService Store
destinations []Destination
queue chan Request
closed uint32 // non-zero means closed. uses uint instead of bool for atomic
ctx context.Context
@@ -27,8 +26,7 @@ type Service struct {
// Destination defines interface for a given destination service, like telegram, email and so on
type Destination interface {
fmt.Stringer
Send(context.Context, Request) error
SendVerification(context.Context, VerificationRequest) error
Send(ctx context.Context, req Request) error
}
// Store defines the minimal interface accessing stored comments used by notifier
@@ -37,18 +35,20 @@ type Store interface {
GetUserEmail(siteID string, userID string) (string, error)
}
// Request notification for a Comment
// Request notification either about comment or about particular user verification
type Request struct {
Comment store.Comment
parent store.Comment
Emails []string
Comment store.Comment // if set sent notifications about new comment
parent store.Comment // fetched only in case Comment is set
Email string // if set (also) send email
ForAdmin bool // if set, message supposed to be sent to administrator
Verification VerificationMetadata // if set sent verification notification
}
// VerificationRequest notification for user
type VerificationRequest struct {
// VerificationMetadata required to send notify method verification message
type VerificationMetadata struct {
SiteID string
User string
Email string // if set, send email only
Token string
}
@@ -62,12 +62,11 @@ func NewService(dataService Store, size int, destinations ...Destination) *Servi
}
ctx, cancel := context.WithCancel(context.Background())
res := Service{
dataService: dataService,
queue: make(chan Request, size),
verificationQueue: make(chan VerificationRequest, size),
destinations: destinations,
ctx: ctx,
cancel: cancel,
dataService: dataService,
queue: make(chan Request, size),
destinations: destinations,
ctx: ctx,
cancel: cancel,
}
if len(destinations) > 0 {
go res.do()
@@ -81,10 +80,18 @@ func (s *Service) Submit(req Request) {
if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
return
}
// parent comment is fetched only if comment is present in the Request
if s.dataService != nil && req.Comment.ParentID != "" {
if p, err := s.dataService.Get(req.Comment.Locator, req.Comment.ParentID, store.User{}); err == nil {
req.parent = p
req.Emails = deduplicateStrings(s.getNotificationEmails(req, p))
// user notification, should fetch email for it.
// administrator notification comes with pre-set email
if req.Email == "" {
req.Email, err = s.dataService.GetUserEmail(req.Comment.Locator.SiteID, p.User.ID)
if err != nil {
log.Printf("[WARN] can't read email for %s, %v", p.User.ID, err)
}
}
}
}
select {
@@ -94,45 +101,11 @@ func (s *Service) Submit(req Request) {
}
}
// getNotificationEmails returns list of emails for notifications for provided comment.
// Emails is not added to the returned list in case original message is from the same user as the notification receiver.
func (s *Service) getNotificationEmails(req Request, notifyComment store.Comment) (result []string) {
// add current user email only if the user is not the one who wrote the original comment
if notifyComment.User.ID != req.Comment.User.ID {
email, err := s.dataService.GetUserEmail(req.Comment.Locator.SiteID, notifyComment.User.ID)
if err != nil {
log.Printf("[WARN] can't read email for %s, %v", notifyComment.User.ID, err)
}
if email != "" {
result = append(result, email)
}
}
if notifyComment.ParentID != "" {
if p, err := s.dataService.Get(req.Comment.Locator, notifyComment.ParentID, store.User{}); err == nil {
result = append(result, s.getNotificationEmails(req, p)...)
}
}
return result
}
// SubmitVerification to internal channel if not busy, drop if can't send
func (s *Service) SubmitVerification(req VerificationRequest) {
if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
return
}
select {
case s.verificationQueue <- req:
default:
log.Printf("[WARN] can't send verification to queue, %s for %s", req.User, req.Email)
}
}
// Close queue channel and wait for completion
func (s *Service) Close() {
if s.queue != nil {
log.Print("[DEBUG] close notifier")
close(s.queue)
close(s.verificationQueue)
s.cancel()
<-s.ctx.Done()
}
@@ -140,60 +113,21 @@ func (s *Service) Close() {
}
func (s *Service) do() {
defer log.Print("[WARN] terminated notifier")
var wg sync.WaitGroup
for {
select {
case c, ok := <-s.queue:
if !ok {
return
}
wg.Add(len(s.destinations))
for _, dest := range s.destinations {
go func(d Destination) {
if err := d.Send(s.ctx, c); err != nil {
log.Printf("[WARN] failed to send to %s, %s", d, err)
}
wg.Done()
}(dest)
}
wg.Wait()
case v, ok := <-s.verificationQueue:
if !ok {
return
}
wg.Add(len(s.destinations))
for _, dest := range s.destinations {
go func(d Destination) {
if err := d.SendVerification(s.ctx, v); err != nil {
log.Printf("[WARN] failed to send to %s, %s", d, err)
}
wg.Done()
}(dest)
}
wg.Wait()
case <-s.ctx.Done():
return
for c := range s.queue {
var wg sync.WaitGroup
wg.Add(len(s.destinations))
for _, dest := range s.destinations {
go func(d Destination) {
if err := d.Send(s.ctx, c); err != nil {
log.Printf("[WARN] failed to send to %s, %s", d, err)
}
wg.Done()
}(dest)
}
wg.Wait()
}
log.Print("[WARN] terminated notifier")
}
// NopService is do-nothing notifier, without destinations
var NopService = &Service{}
// deduplicateStrings returns provided slice of strings will all duplicates removed.
// Resulting slice is not sorted.
func deduplicateStrings(source []string) []string {
set := make(map[string]struct{})
for _, k := range source {
set[k] = struct{}{}
}
result := make([]string, 0, len(set))
for k := range set {
result = append(result, k)
}
return result
}
+4 -30
View File
@@ -11,11 +11,10 @@ import (
// MockDest is a destination mock
type MockDest struct {
data []Request
verificationData []VerificationRequest
id int
closed bool
lock sync.Mutex
data []Request
id int
closed bool
lock sync.Mutex
}
// Send mock
@@ -33,21 +32,6 @@ func (m *MockDest) Send(ctx context.Context, r Request) error {
return nil
}
// SendVerification mock
func (m *MockDest) SendVerification(ctx context.Context, v VerificationRequest) error {
m.lock.Lock()
defer m.lock.Unlock()
select {
case <-time.After(10 * time.Millisecond):
m.verificationData = append(m.verificationData, v)
log.Printf("sent verification %s -> %d", v.User, m.id)
case <-ctx.Done():
log.Printf("verification ctx closed %d", m.id)
m.closed = true
}
return nil
}
// Get mock
func (m *MockDest) Get() []Request {
m.lock.Lock()
@@ -56,14 +40,4 @@ func (m *MockDest) Get() []Request {
copy(res, m.data)
return res
}
// GetVerify mock
func (m *MockDest) GetVerify() []VerificationRequest {
m.lock.Lock()
defer m.lock.Unlock()
res := make([]VerificationRequest, len(m.verificationData))
copy(res, m.verificationData)
return res
}
func (m *MockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
+8 -178
View File
@@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestService_NoDestinations(t *testing.T) {
@@ -52,42 +52,15 @@ func TestService_WithDrops(t *testing.T) {
s.Submit(Request{Comment: store.Comment{ID: "100"}})
s.Submit(Request{Comment: store.Comment{ID: "101"}})
time.Sleep(time.Millisecond * 11)
s.Submit(Request{Comment: store.Comment{ID: "102"}})
time.Sleep(time.Millisecond * 21)
time.Sleep(time.Millisecond * 11)
s.Close()
s.Submit(Request{Comment: store.Comment{ID: "111"}}) // safe to send after close
assert.LessOrEqual(t, len(d1.Get()), 2, "at least one comment from three dropped from d1, got: %v", d1.Get())
assert.LessOrEqual(t, len(d2.Get()), 2, "at least one comment from three dropped from d2, got: %v", d2.Get())
}
func TestService_SubmitVerificationWithDrops(t *testing.T) {
d1, d2 := &MockDest{id: 1}, &MockDest{id: 2}
s := NewService(nil, 1, d1, d2)
assert.NotNil(t, s)
s.SubmitVerification(VerificationRequest{
SiteID: "remark",
User: "testUser",
Email: "test@example.org",
Token: "testToken",
})
s.SubmitVerification(VerificationRequest{})
s.SubmitVerification(VerificationRequest{})
time.Sleep(time.Millisecond * 21)
s.Close()
s.SubmitVerification(VerificationRequest{}) // safe to send after close
assert.LessOrEqual(t, len(d2.GetVerify()), 2, "one request from three dropped from d2, got: %v", d2.GetVerify())
verifyDest := d1.GetVerify()
require.LessOrEqual(t, len(verifyDest), 2, "one request from three dropped from d1, got: %v", verifyDest)
assert.Equal(t, "remark", verifyDest[0].SiteID)
assert.Equal(t, "testUser", verifyDest[0].User)
assert.Equal(t, "test@example.org", verifyDest[0].Email)
assert.Equal(t, "testToken", verifyDest[0].Token)
assert.Equal(t, 2, len(d1.Get()), "one comment from three dropped from d1, got: %v", d1.Get())
assert.Equal(t, 2, len(d2.Get()), "one comment from three dropped from d2, got: %v", d2.Get())
}
func TestService_Many(t *testing.T) {
@@ -97,20 +70,16 @@ func TestService_Many(t *testing.T) {
for i := 0; i < 10; i++ {
s.Submit(Request{Comment: store.Comment{ID: fmt.Sprintf("%d", 100+i)}})
s.SubmitVerification(VerificationRequest{User: fmt.Sprintf("%d", 100+i)})
time.Sleep(time.Millisecond * time.Duration(rand.Int31n(20)))
}
s.Close()
time.Sleep(time.Millisecond * 10)
assert.NotEqual(t, 10, len(d1.Get()), "some comments dropped from d1")
assert.NotEqual(t, 10, len(d1.GetVerify()), "some verifications dropped from d1")
assert.NotEqual(t, 10, len(d2.Get()), "some comments dropped from d2")
assert.NotEqual(t, 10, len(d2.GetVerify()), "some verifications dropped from d2")
assert.True(t, d1.closed)
assert.True(t, d2.closed)
assert.Equal(t, "mock id=1, closed=true", d1.String())
}
func TestService_WithParent(t *testing.T) {
@@ -137,138 +106,6 @@ func TestService_WithParent(t *testing.T) {
assert.Equal(t, "", destRes[1].parent.ID)
}
func TestService_EmailRetrieval(t *testing.T) {
dest := &MockDest{id: 1}
dataStore := &mockStore{data: map[string]store.Comment{}, emailData: map[string]string{}}
dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u1"}}
dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p1", User: store.User{ID: "u2"}}
dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
dataStore.emailData["u1"] = "u1@example.com"
s := NewService(dataStore, 1, dest)
assert.NotNil(t, s)
// one comment, one notification
s.Submit(Request{Comment: dataStore.data["p1"]})
time.Sleep(time.Millisecond * 110)
destRes := dest.Get()
require.Equal(t, 1, len(destRes), "one comment notified")
assert.Equal(t, "p1", destRes[0].Comment.ID)
assert.Empty(t, destRes[0].parent)
assert.Empty(t, destRes[0].Emails)
// reply to the first comment, same comment as one in original comment
s.Submit(Request{Comment: dataStore.data["p2"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 2, len(destRes), "two comment notified")
assert.Equal(t, "p2", destRes[1].Comment.ID)
assert.Equal(t, "p1", destRes[1].parent.ID)
assert.Equal(t, "u1", destRes[1].parent.User.ID)
assert.Empty(t, destRes[1].Emails, "u1 is not notified they are the one who left the comment")
// another reply to the first comment, another user
s.Submit(Request{Comment: dataStore.data["p3"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 3, len(destRes), "three comment notified")
assert.Equal(t, "p3", destRes[2].Comment.ID)
assert.Equal(t, "p1", destRes[2].parent.ID)
assert.Equal(t, "u1", destRes[2].parent.User.ID)
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
// reply to the last comment by another user, should trigger email retrieval error
s.Submit(Request{Comment: dataStore.data["p4"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 4, len(destRes), "four comment notified")
assert.Equal(t, "p4", destRes[3].Comment.ID)
assert.Equal(t, "p3", destRes[3].parent.ID)
assert.Equal(t, "u2", destRes[3].parent.User.ID)
assert.Empty(t, destRes[3].Emails, "no email can be retrieved for u2")
s.Close()
}
func TestService_Recursive(t *testing.T) {
dest := &MockDest{id: 1}
dataStore := &mockStore{data: map[string]store.Comment{}, emailData: map[string]string{}}
dataStore.data["p1"] = store.Comment{ID: "p1", User: store.User{ID: "u1"}}
dataStore.data["p2"] = store.Comment{ID: "p2", ParentID: "p1", User: store.User{ID: "u2"}}
dataStore.data["p3"] = store.Comment{ID: "p3", ParentID: "p2", User: store.User{ID: "u3"}}
dataStore.data["p4"] = store.Comment{ID: "p4", ParentID: "p3", User: store.User{ID: "u1"}}
dataStore.data["p5"] = store.Comment{ID: "p5", ParentID: "p4", User: store.User{ID: "u4"}}
dataStore.emailData["u1"] = "u1@example.com"
// second comment goes without email address for notification
dataStore.emailData["u3"] = "u3@example.com"
s := NewService(dataStore, 1, dest)
assert.NotNil(t, s)
// one comment from u1 with email set
s.Submit(Request{Comment: dataStore.data["p1"]})
time.Sleep(time.Millisecond * 110)
destRes := dest.Get()
require.Equal(t, 1, len(destRes), "one comment notified")
assert.Equal(t, "p1", destRes[0].Comment.ID)
assert.Empty(t, destRes[0].parent)
assert.Empty(t, destRes[0].Emails)
// reply to the first comment from u2 without email set
s.Submit(Request{Comment: dataStore.data["p2"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 2, len(destRes), "two comment notified")
assert.Equal(t, "p2", destRes[1].Comment.ID)
assert.Equal(t, "p1", destRes[1].parent.ID)
assert.Equal(t, "u1", destRes[1].parent.User.ID)
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[1].Emails)
// reply to the second comment from u3 with email set
s.Submit(Request{Comment: dataStore.data["p3"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 3, len(destRes), "three comment notified")
assert.Equal(t, "p3", destRes[2].Comment.ID)
assert.Equal(t, "p2", destRes[2].parent.ID)
assert.Equal(t, "u2", destRes[2].parent.User.ID)
assert.ElementsMatch(t, []string{"u1@example.com"}, destRes[2].Emails)
// reply to the third comment from u1 (author of the first comment), only u3 should be notified
s.Submit(Request{Comment: dataStore.data["p4"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 4, len(destRes), "four comment notified once each")
assert.Equal(t, "p4", destRes[3].Comment.ID)
assert.Equal(t, "p3", destRes[3].parent.ID)
assert.Equal(t, "u3", destRes[3].parent.User.ID)
assert.ElementsMatch(t, []string{"u3@example.com"}, destRes[3].Emails, "u1 is not notified they are the one who left the comment")
// reply to the fourth comment from u4, u1 and u3 should be notified once as a result
s.Submit(Request{Comment: dataStore.data["p5"]})
time.Sleep(time.Millisecond * 110)
destRes = dest.Get()
require.Equal(t, 5, len(destRes), "four comment notified once each")
assert.Equal(t, "p5", destRes[4].Comment.ID)
assert.Equal(t, "p4", destRes[4].parent.ID)
assert.Equal(t, "u1", destRes[4].parent.User.ID)
assert.ElementsMatch(t, []string{"u1@example.com", "u3@example.com"}, destRes[4].Emails, "u3 and u1 notified once")
s.Close()
}
func TestService_Nop(t *testing.T) {
s := NopService
s.Submit(Request{Comment: store.Comment{}})
@@ -276,10 +113,7 @@ func TestService_Nop(t *testing.T) {
assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
}
type mockStore struct {
data map[string]store.Comment
emailData map[string]string
}
type mockStore struct{ data map[string]store.Comment }
func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment, error) {
res, ok := m.data[id]
@@ -289,10 +123,6 @@ func (m mockStore) Get(_ store.Locator, id string, _ store.User) (store.Comment,
return res, nil
}
func (m mockStore) GetUserEmail(_, userID string) (string, error) {
email, ok := m.emailData[userID]
if !ok {
return "", errors.New("no such user")
}
return email, nil
func (m mockStore) GetUserEmail(_ string, _ string) (string, error) {
return "", errors.New("no such user")
}
+12 -17
View File
@@ -8,7 +8,6 @@ import (
"html"
"net/http"
"strconv"
"strings"
"time"
log "github.com/go-pkgz/lgr"
@@ -28,7 +27,8 @@ const telegramTimeOut = 5000 * time.Millisecond
const telegramAPIPrefix = "https://api.telegram.org/bot"
// NewTelegram makes telegram bot for notifications
func NewTelegram(token, channelID string, timeout time.Duration, api string) (*Telegram, error) {
func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
channelID = "@" + channelID // if channelID not a number enforce @ prefix
}
@@ -86,6 +86,15 @@ func NewTelegram(token, channelID string, timeout time.Duration, api string) (*T
// Send to telegram channel
func (t *Telegram) Send(ctx context.Context, req Request) error {
if req.Comment.ID == "" {
// verification request received, send nothing
return nil
}
if req.ForAdmin {
// request for administrator received, do nothing with it
// as we already sent message on request without this flag set
return nil
}
client := http.Client{Timeout: telegramTimeOut}
log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.Comment.ID)
@@ -96,7 +105,7 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
from = "*" + from + "*"
link := fmt.Sprintf("↦ [original comment](%s)", req.Comment.Locator.URL+uiNav+req.Comment.ID)
if req.Comment.PostTitle != "" {
link = fmt.Sprintf("↦ [%s](%s)", t.escapeTitle(req.Comment.PostTitle), req.Comment.Locator.URL+uiNav+req.Comment.ID)
link = fmt.Sprintf("↦ [%s](%s)", req.Comment.PostTitle, req.Comment.Locator.URL+uiNav+req.Comment.ID)
}
u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
t.apiPrefix, t.token, t.channelID)
@@ -143,20 +152,6 @@ func (t *Telegram) Send(ctx context.Context, req Request) error {
return nil
}
func (t *Telegram) escapeTitle(title string) string {
escSymbols := []string{"[", "]", "(", ")"}
res := title
for _, esc := range escSymbols {
res = strings.Replace(res, esc, "\\"+esc, -1)
}
return res
}
// SendVerification is not implemented for telegram
func (t *Telegram) SendVerification(_ context.Context, _ VerificationRequest) error {
return nil
}
func (t *Telegram) String() string {
return "telegram: " + t.channelID
}
+2 -43
View File
@@ -4,7 +4,6 @@ import (
"context"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
@@ -12,7 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestTelegram_New(t *testing.T) {
@@ -72,12 +71,6 @@ func TestTelegram_Send(t *testing.T) {
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
assert.NoError(t, err)
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
assert.NoError(t, err)
c.PostTitle = "[test title]"
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
assert.NoError(t, err)
tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
assert.Error(t, err, "should failed")
err = tb.Send(context.TODO(), Request{Comment: c, parent: cp})
@@ -85,18 +78,7 @@ func TestTelegram_Send(t *testing.T) {
assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
assert.Equal(t, "telegram: @remark_test", tb.String())
}
func TestTelegram_SendVerification(t *testing.T) {
ts := mockTelegramServer()
defer ts.Close()
tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
assert.NoError(t, err)
assert.NotNil(t, tb)
err = tb.SendVerification(context.TODO(), VerificationRequest{})
assert.NoError(t, err)
require.NoError(t, tb.Send(context.TODO(), Request{}), "Empty Comment doesn't send anything")
}
func mockTelegramServer() *httptest.Server {
@@ -141,26 +123,3 @@ func mockTelegramServer() *httptest.Server {
return httptest.NewServer(router)
}
func TestTelegram_escapeTitle(t *testing.T) {
tbl := []struct {
inp string
out string
}{
{"", ""},
{"something 123", "something 123"},
{"something [123]", "something \\[123\\]"},
{"something (123)", "something \\(123\\)"},
{"something (123) [aaa]", "something \\(123\\) \\[aaa\\]"},
}
tb := Telegram{}
for i, tt := range tbl {
tt := tt
t.Run(strconv.Itoa(i), func(t *testing.T) {
assert.Equal(t, tt.out, tb.escapeTitle(tt.inp))
})
}
}
-1
View File
@@ -1 +0,0 @@
{{
-20
View File
@@ -1,20 +0,0 @@
{{- if .ForAdmin}}
New comment from {{.UserName}} on your site {{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
{{- else }}
New reply from {{.UserName}} on your comment{{if .PostTitle}} to «{{.PostTitle}}»{{ end }}
{{- end }}
{{- if .ParentCommentText}}
{{.ParentUserPicture}}
{{.ParentUserName}}
{{.ParentCommentDate.Format "02.01.2006 at 15:04"}}
Parent comment link: {{.ParentCommentLink}}
{{.ParentCommentText}}
{{- end }}
User: {{.UserName}}
{{.CommentDate.Format "02.01.2006 at 15:04"}}
Comment: {{.CommentText}}
{{.Email}} {{if not .ForAdmin}} for {{.ParentUserName}}{{ end }}
{{- if .UnsubscribeLink}}
Unsubscribe link: {{.UnsubscribeLink}}
{{- end }}
-7
View File
@@ -1,7 +0,0 @@
Confirmation for {{.User}} on site {{.Site}}
{{- if .SubscribeURL}}
Subscribe url: {{.SubscribeURL}}{{.Token}}
{{- end }}
Token:{{.Token}}
Sent to {{.Email}}
+3 -3
View File
@@ -13,9 +13,9 @@ import (
log "github.com/go-pkgz/lgr"
R "github.com/go-pkgz/rest"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
)
// admin provides router for all requests available for admin users only
+5 -27
View File
@@ -17,11 +17,12 @@ import (
"github.com/go-pkgz/auth/token"
cache "github.com/go-pkgz/lcw"
R "github.com/go-pkgz/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
func TestAdmin_Delete(t *testing.T) {
@@ -47,9 +48,9 @@ func TestAdmin_Delete(t *testing.T) {
// check multi count
resp, err := post(t, ts.URL+"/api/v1/counts?site=remark42", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
bb, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, resp.Body.Close())
assert.NoError(t, err)
j := []store.PostInfo{}
err = json.Unmarshal(bb, &j)
@@ -61,10 +62,10 @@ func TestAdmin_Delete(t *testing.T) {
req, err := http.NewRequest(http.MethodDelete,
fmt.Sprintf("%s/api/v1/admin/comment/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
require.NoError(t, err)
defer resp.Body.Close()
requireAdminOnly(t, req)
resp, err = sendReq(t, req, adminUmputunToken)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
@@ -98,7 +99,6 @@ func TestAdmin_Delete(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
bb, err = ioutil.ReadAll(resp.Body)
assert.NoError(t, resp.Body.Close())
assert.NoError(t, err)
j = []store.PostInfo{}
err = json.Unmarshal(bb, &j)
@@ -141,7 +141,6 @@ func TestAdmin_Title(t *testing.T) {
requireAdminOnly(t, req)
resp, err := sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=%s/post1", ts.URL, id1, tss.URL))
@@ -176,7 +175,6 @@ func TestAdmin_DeleteUser(t *testing.T) {
requireAdminOnly(t, req)
resp, err := sendReq(t, req, adminUmputunToken)
assert.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
// all 3 comments here, but for id2 they deleted
@@ -226,7 +224,6 @@ func TestAdmin_Pin(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
return resp.StatusCode
}
@@ -383,7 +380,6 @@ func TestAdmin_BlockedList(t *testing.T) {
assert.NoError(t, err)
res, err := sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, res.Body.Close())
assert.Equal(t, 200, res.StatusCode)
// block user2
@@ -392,7 +388,6 @@ func TestAdmin_BlockedList(t *testing.T) {
assert.NoError(t, err)
res, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, res.Body.Close())
assert.Equal(t, 200, res.StatusCode)
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=remark42", nil)
@@ -403,7 +398,6 @@ func TestAdmin_BlockedList(t *testing.T) {
users := []store.BlockedUser{}
err = json.NewDecoder(res.Body).Decode(&users)
assert.NoError(t, err)
require.NoError(t, res.Body.Close())
require.Equal(t, 2, len(users), "two users blocked")
assert.Equal(t, "user1", users[0].ID)
assert.Equal(t, "user1 name", users[0].Name)
@@ -420,7 +414,6 @@ func TestAdmin_BlockedList(t *testing.T) {
users = []store.BlockedUser{}
err = json.NewDecoder(res.Body).Decode(&users)
assert.NoError(t, err)
require.NoError(t, res.Body.Close())
assert.Equal(t, 1, len(users), "one user left blocked")
}
@@ -448,11 +441,9 @@ func TestAdmin_ReadOnly(t *testing.T) {
assert.NoError(t, err)
resp, err := sendReq(t, req, "") // non-admin user
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 401, resp.StatusCode)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
assert.NoError(t, err)
@@ -467,7 +458,6 @@ func TestAdmin_ReadOnly(t *testing.T) {
require.NoError(t, err)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
// reset post's read-only
@@ -476,7 +466,6 @@ func TestAdmin_ReadOnly(t *testing.T) {
assert.NoError(t, err)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
assert.NoError(t, err)
@@ -491,7 +480,6 @@ func TestAdmin_ReadOnly(t *testing.T) {
require.NoError(t, err)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusCreated, resp.StatusCode)
}
@@ -506,7 +494,6 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
requireAdminOnly(t, req)
resp, err := sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
_, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
assert.Error(t, err)
@@ -542,7 +529,6 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
requireAdminOnly(t, req)
resp, err := sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
assert.NoError(t, err)
@@ -554,7 +540,6 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
assert.NoError(t, err)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 403, resp.StatusCode)
info, err = srv.DataService.Info(store.Locator{SiteID: "remark42", URL: "https://radio-t.com/blah"}, 0)
assert.NoError(t, err)
@@ -584,7 +569,6 @@ func TestAdmin_Verify(t *testing.T) {
requireAdminOnly(t, req)
resp, err := sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
verified = srv.DataService.IsVerified("remark42", "user1")
assert.True(t, verified)
@@ -603,7 +587,6 @@ func TestAdmin_Verify(t *testing.T) {
assert.NoError(t, err)
resp, err = sendReq(t, req, adminUmputunToken)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
verified = srv.DataService.IsVerified("remark42", "user1")
assert.False(t, verified)
@@ -724,7 +707,6 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
_, err = srv.DataService.User("remark42", "user1", 0, 0, store.User{})
@@ -757,7 +739,6 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, 400, resp.StatusCode)
// try with bad auth
@@ -785,7 +766,6 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
req.SetBasicAuth("admin", "bad-password")
resp, err = client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, 403, resp.StatusCode)
// try bad user
@@ -798,7 +778,6 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err = client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, 400, resp.StatusCode, resp.Status)
// try without deleteme flag
@@ -814,7 +793,6 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
assert.Equal(t, 403, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.True(t, strings.Contains(string(b), "can't use provided token"))
}
+4 -4
View File
@@ -17,8 +17,8 @@ import (
R "github.com/go-pkgz/rest"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/migrator"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
)
// Migrator rest with import and export controllers
@@ -37,7 +37,7 @@ type Migrator struct {
// KeyStore defines sub-interface for consumers needed just a key
type KeyStore interface {
Key(siteID string) (key string, err error)
Key() (key string, err error)
}
// POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
@@ -212,7 +212,7 @@ func (m *Migrator) remapCtrl(w http.ResponseWriter, r *http.Request) {
}
// runImport reads from tmpfile and import for given siteID and provider
func (m *Migrator) runImport(siteID, provider, tmpfile string) {
func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
m.setBusy(siteID, true)
defer func() {
+2 -13
View File
@@ -17,8 +17,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_Import(t *testing.T) {
@@ -124,7 +124,6 @@ func TestMigrator_ImportRejected(t *testing.T) {
assert.NoError(t, err)
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
@@ -148,7 +147,6 @@ func TestMigrator_ImportDouble(t *testing.T) {
assert.NoError(t, err)
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
client = &http.Client{Timeout: 5 * time.Second}
@@ -158,7 +156,6 @@ func TestMigrator_ImportDouble(t *testing.T) {
assert.NoError(t, err)
resp, err = client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusConflict, resp.StatusCode)
waitForMigrationCompletion(t, ts)
}
@@ -184,7 +181,6 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
require.NoError(t, err)
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusAccepted, resp.StatusCode)
client = &http.Client{Timeout: 5 * time.Second}
@@ -194,7 +190,6 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
assert.NoError(t, err)
resp, err = client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
waitForMigrationCompletion(t, ts)
@@ -220,7 +215,6 @@ func TestMigrator_Export(t *testing.T) {
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusAccepted, resp.StatusCode)
waitForMigrationCompletion(t, ts)
@@ -237,7 +231,6 @@ func TestMigrator_Export(t *testing.T) {
assert.NoError(t, err)
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
t.Logf("%s", string(ungzBody))
@@ -253,7 +246,6 @@ func TestMigrator_Export(t *testing.T) {
body, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 3, strings.Count(string(body), "\n"))
assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
t.Logf("%s", string(body))
@@ -262,7 +254,6 @@ func TestMigrator_Export(t *testing.T) {
require.NoError(t, err)
resp, err = client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
@@ -314,7 +305,6 @@ func TestMigrator_Remap(t *testing.T) {
rules := "https://remark42.com/* https://www.remark42.com/*"
resp, err := post(t, ts.URL+"/api/v1/admin/remap?site=remark42", rules) // auth as admin
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusAccepted, resp.StatusCode)
waitForMigrationCompletion(t, ts)
@@ -362,7 +352,6 @@ func TestMigrator_RemapReject(t *testing.T) {
require.NoError(t, err)
resp, err := client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
+35 -53
View File
@@ -6,11 +6,12 @@ import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/didip/tollbooth/v6"
"github.com/didip/tollbooth"
"github.com/didip/tollbooth_chi"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
@@ -24,13 +25,12 @@ import (
"github.com/pkg/errors"
"github.com/rakyll/statik/fs"
"github.com/umputun/remark42/backend/app/notify"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/rest/proxy"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark42/backend/app/templates"
"github.com/umputun/remark/backend/app/notify"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
// Rest is a rest access server
@@ -45,10 +45,12 @@ type Rest struct {
Migrator *Migrator
NotifyService *notify.Service
ImageService *image.Service
Streamer *Streamer
AnonVote bool
WebRoot string
RemarkURL string
AdminEmail string
ReadOnlyAge int
SharedSecret string
ScoreThresholds struct {
@@ -59,9 +61,6 @@ type Rest struct {
EmailNotifications bool
EmojiEnabled bool
SimpleView bool
ProxyCORS bool
SendJWTHeader bool
AllowedAncestors []string // sets Content-Security-Policy "frame-ancestors ..."
SSLConfig SSLConfig
httpsServer *http.Server
@@ -78,7 +77,6 @@ type Rest struct {
type LoadingCache interface {
Get(key lcw.Key, fn func() ([]byte, error)) (data []byte, err error) // load from cache if found or put to cache and return
Flush(req lcw.FlusherRequest) // evict matched records
Close() error
}
const hardBodyLimit = 1024 * 64 // limit size of body
@@ -187,24 +185,15 @@ func (s *Rest) routes() chi.Router {
s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
if s.ProxyCORS {
log.Printf("[WARN] internal CORS disabled")
} else {
corsMiddleware := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
ExposedHeaders: []string{"Authorization"},
AllowCredentials: true,
MaxAge: 300,
})
router.Use(corsMiddleware.Handler)
}
if len(s.AllowedAncestors) > 0 {
log.Printf("[INFO] allowed from %+v only", s.AllowedAncestors)
router.Use(frameAncestors(s.AllowedAncestors))
}
corsMiddleware := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
ExposedHeaders: []string{"Authorization"},
AllowCredentials: true,
MaxAge: 300,
})
router.Use(corsMiddleware.Handler)
ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
@@ -213,13 +202,13 @@ func (s *Rest) routes() chi.Router {
router.Group(func(r chi.Router) {
r.Use(middleware.Timeout(5 * time.Second))
r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)), middleware.NoCache)
r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
r.Mount("/auth", authHandler)
})
router.Group(func(r chi.Router) {
r.Use(middleware.Timeout(5 * time.Second))
r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
r.Mount("/avatar", avatarHandler)
})
@@ -231,6 +220,7 @@ func (s *Rest) routes() chi.Router {
rapi.Group(func(rava chi.Router) {
rava.Use(middleware.Timeout(5 * time.Second))
rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
rava.Use(middleware.NoCache)
rava.Mount("/avatar", avatarHandler)
})
@@ -259,6 +249,14 @@ func (s *Rest) routes() chi.Router {
})
// open routes, streams, no send timeout
rapi.Route("/stream", func(rstream chi.Router) {
rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
rstream.Get("/info", s.pubRest.infoStreamCtrl)
rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
})
// open routes, cached
rapi.Group(func(ropen chi.Router) {
ropen.Use(middleware.Timeout(30 * time.Second))
@@ -307,7 +305,8 @@ func (s *Rest) routes() chi.Router {
rauth.Use(middleware.Timeout(10 * time.Second))
rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
rauth.Use(authMiddleware.Auth, matchSiteID)
rauth.Use(middleware.NoCache, logInfoWithBody)
rauth.Use(middleware.NoCache)
rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
rauth.Post("/comment", s.privRest.createCommentCtrl)
@@ -354,6 +353,7 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
commentFormatter: s.CommentFormatter,
readOnlyAge: s.ReadOnlyAge,
webRoot: s.WebRoot,
streamer: s.Streamer,
}
privGrp := private{
@@ -365,8 +365,8 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
authenticator: s.Authenticator,
notifyService: s.NotifyService,
remarkURL: s.RemarkURL,
adminEmail: s.AdminEmail,
anonVote: s.AnonVote,
templates: templates.NewFS(),
}
admGrp := admin{
@@ -417,7 +417,6 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
EmailNotifications bool `json:"email_notifications"`
EmojiEnabled bool `json:"emoji_enabled"`
SimpleView bool `json:"simple_view"`
SendJWTHeader bool `json:"send_jwt_header"`
}{
Version: s.Version,
EditDuration: int(s.DataService.EditDuration.Seconds()),
@@ -433,7 +432,6 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
EmojiEnabled: s.EmojiEnabled,
AnonVote: s.AnonVote,
SimpleView: s.SimpleView,
SendJWTHeader: s.SendJWTHeader,
}
cnf.Auth = []string{}
@@ -585,7 +583,7 @@ func cacheControl(expiration time.Duration, version string) func(http.Handler) h
fn := func(w http.ResponseWriter, r *http.Request) {
e := `"` + etag(r, version) + `"`
w.Header().Set("Etag", e)
w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, no-cache", int(expiration.Seconds())))
w.Header().Set("Cache-Control", "max-age="+strconv.Itoa(int(expiration.Seconds())))
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, e) {
@@ -599,22 +597,6 @@ func cacheControl(expiration time.Duration, version string) func(http.Handler) h
}
}
// frameAncestors is a middleware setting Content-Security-Policy "frame-ancestors host1 host2 ..."
// prevents loading of comments widgets from any other origins. In case if the list of allowed empty, ignored.
func frameAncestors(hosts []string) func(http.Handler) http.Handler {
return func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
if len(hosts) == 0 {
h.ServeHTTP(w, r)
return
}
w.Header().Set("Content-Security-Policy", "frame-ancestors "+strings.Join(hosts, " ")+";")
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
func parseError(err error, defaultCode int) (code int) {
code = defaultCode
+47 -41
View File
@@ -22,13 +22,12 @@ import (
R "github.com/go-pkgz/rest"
"github.com/hashicorp/go-multierror"
"github.com/umputun/remark42/backend/app/notify"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark42/backend/app/templates"
"github.com/umputun/remark/backend/app/notify"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
type private struct {
@@ -40,8 +39,8 @@ type private struct {
notifyService *notify.Service
authenticator *auth.Service
remarkURL string
adminEmail string
anonVote bool
templates templates.FileReader
}
type privStore interface {
@@ -60,6 +59,21 @@ type privStore interface {
Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
}
const unsubscribeHTML = `<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Successfully unsubscribed</p>
</div>
</body>
</html>
`
// POST /comment - adds comment, resets all immutable fields
func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
@@ -118,9 +132,14 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
// user notification
if s.notifyService != nil {
s.notifyService.Submit(notify.Request{Comment: finalComment})
}
// admin notification
if s.notifyService != nil && s.adminEmail != "" {
s.notifyService.Submit(notify.Request{Comment: finalComment, Email: s.adminEmail, ForAdmin: true})
}
log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -263,8 +282,7 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
address := r.URL.Query().Get("address")
siteID := r.URL.Query().Get("site")
if address == "" {
rest.SendErrorJSON(w, r, http.StatusBadRequest,
errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
rest.SendErrorJSON(w, r, http.StatusBadRequest, errors.New("missing parameter"), "address parameter is required", rest.ErrInternal)
return
}
existingAddress, err := s.dataService.GetUserEmail(siteID, user.ID)
@@ -272,8 +290,7 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
log.Printf("[WARN] can't read email for %s, %v", user.ID, err)
}
if address == existingAddress {
rest.SendErrorJSON(w, r, http.StatusConflict,
errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already verified"), "email address is already verified for this user", rest.ErrInternal)
return
}
claims := token.Claims{
@@ -292,12 +309,14 @@ func (s *private) sendEmailConfirmationCtrl(w http.ResponseWriter, r *http.Reque
return
}
s.notifyService.SubmitVerification(
notify.VerificationRequest{
SiteID: siteID,
User: user.Name,
Email: address,
Token: tkn,
s.notifyService.Submit(
notify.Request{
Email: address,
Verification: notify.VerificationMetadata{
SiteID: siteID,
User: user.Name,
Token: tkn,
},
},
)
@@ -360,28 +379,25 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request)
func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
tkn := r.URL.Query().Get("tkn")
if tkn == "" {
rest.SendErrorHTML(w, r, http.StatusBadRequest,
errors.New("missing parameter"), "token parameter is required", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("missing parameter"), "token parameter is required", rest.ErrInternal)
return
}
siteID := r.URL.Query().Get("site")
confClaims, err := s.authenticator.TokenService().Parse(tkn)
if err != nil {
rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal)
return
}
if s.authenticator.TokenService().IsExpired(confClaims) {
rest.SendErrorHTML(w, r, http.StatusForbidden,
errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusForbidden, errors.New("expired"), "failed to verify confirmation token", rest.ErrInternal)
return
}
elems := strings.Split(confClaims.Handshake.ID, "::")
if len(elems) != 2 {
rest.SendErrorHTML(w, r, http.StatusBadRequest,
errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New(confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal)
return
}
userID := elems[0]
@@ -392,14 +408,11 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[WARN] can't read email for %s, %v", userID, err)
}
if existingAddress == "" {
rest.SendErrorHTML(w, r, http.StatusConflict,
errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusConflict, errors.New("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal)
return
}
if address != existingAddress {
rest.SendErrorHTML(w, r, http.StatusBadRequest,
errors.New("wrong email unsubscription"), "email address in request does not match known for this user",
rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusBadRequest, errors.New("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal)
return
}
@@ -407,7 +420,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil {
code := parseError(err, rest.ErrInternal)
rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code, s.templates)
rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code)
return
}
// clean User.Email from the token, if user has the token
@@ -418,7 +431,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
if claims.User != nil && claims.User.Email != "" {
claims.User.Email = ""
if _, err = s.authenticator.TokenService().Set(w, claims); err != nil {
rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal, s.templates)
rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal)
return
}
}
@@ -429,15 +442,8 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) {
panic(err)
}
}
MustRead := func(path string) string {
file, err := s.templates.ReadFile(path)
if err != nil {
panic(err)
}
return string(file)
}
tmplstr := MustRead("unsubscribe.html.tmpl")
tmpl := template.Must(template.New("unsubscribe").Parse(tmplstr))
tmpl := template.Must(template.New("unsubscribe").Parse(unsubscribeHTML))
msg := bytes.Buffer{}
MustExecute(tmpl, &msg, nil)
render.HTML(w, r, msg.String())
+31 -87
View File
@@ -3,7 +3,6 @@ package api
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"fmt"
@@ -24,9 +23,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/notify"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/notify"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/image"
)
// gopher png for test, from https://golang.org/src/image/png/example_test.go
@@ -72,7 +71,6 @@ func TestRest_CreateOldPost(t *testing.T) {
resp, err := post(t, ts.URL+"/api/v1/comment",
`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusCreated, resp.StatusCode)
assert.NoError(t, srv.DataService.DeleteAll("remark42"))
@@ -85,7 +83,6 @@ func TestRest_CreateOldPost(t *testing.T) {
resp, err = post(t, ts.URL+"/api/v1/comment",
`{"text": "test 123", "locator":{"site": "remark42","url": "https://radio-t.com/blah1"}}`)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
}
@@ -123,8 +120,8 @@ func TestRest_CreateWithRestrictedWord(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
badComment := `{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1",
"site": "remark42"}}`
badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1",
"site": "remark42"}}`)
resp, err := post(t, ts.URL+"/api/v1/comment", badComment)
assert.NoError(t, err)
@@ -147,7 +144,6 @@ func TestRest_CreateRejected(t *testing.T) {
// try to create without auth
resp, err := http.Post(ts.URL+"/api/v1/comment", "", strings.NewReader(body))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 401, resp.StatusCode)
// try with wrong aud
@@ -157,26 +153,9 @@ func TestRest_CreateRejected(t *testing.T) {
req.Header.Add("X-JWT", devTokenBadAud)
resp, err = client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusForbidden, resp.StatusCode, "reject wrong aud")
}
func TestRest_CreateWithLazyImage(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
body := `{"text": "test 123 ![](http://example.com/image.png)", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`
// create comment
resp, err := post(t, ts.URL+"/api/v1/comment", body)
require.NoError(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
c := store.Comment{}
err = json.Unmarshal(b, &c)
assert.NoError(t, err)
assert.Equal(t, c.Text, "<p>test 123 <img src=\"http://example.com/image.png\" alt=\"\" loading=\"lazy\"/></p>\n")
}
func TestRest_CreateAndGet(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
@@ -333,7 +312,6 @@ func TestRest_UpdateNotOwner(t *testing.T) {
assert.NoError(t, err)
body, err := ioutil.ReadAll(b.Body)
assert.NoError(t, err)
assert.NoError(t, b.Body.Close())
assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
@@ -344,7 +322,6 @@ func TestRest_UpdateNotOwner(t *testing.T) {
req.Header.Add("X-JWT", devToken)
b, err = client.Do(req)
assert.NoError(t, err)
assert.NoError(t, b.Body.Close())
assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
}
@@ -363,7 +340,6 @@ func TestRest_UpdateWrongAud(t *testing.T) {
req.Header.Add("X-JWT", devTokenBadAud)
b, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, b.Body.Close())
assert.Equal(t, http.StatusForbidden, b.StatusCode, "reject update with wrong aut in jwt")
}
@@ -412,7 +388,6 @@ func TestRest_Vote(t *testing.T) {
req.Header.Add("X-JWT", devToken)
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
return resp.StatusCode
}
@@ -425,8 +400,7 @@ func TestRest_Vote(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1, cr.Score)
assert.Equal(t, 1, cr.Vote)
assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
assert.Equal(t, map[string]bool(nil), cr.Votes)
assert.Equal(t, 200, vote(-1), "opposite vote allowed")
body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1))
@@ -462,8 +436,7 @@ func TestRest_Vote(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, -1, cr.Score)
assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
assert.Equal(t, map[string]bool(nil), cr.Votes)
req, err := http.NewRequest("GET",
fmt.Sprintf("%s/api/v1/id/%s?site=remark42&url=https://radio-t.com/blah", ts.URL, id1), nil)
@@ -476,8 +449,7 @@ func TestRest_Vote(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, -1, cr.Score)
assert.Equal(t, 0, cr.Vote, "no vote info for different user")
assert.Equal(t, map[string]bool(nil), cr.Votes, "hidden")
assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs, "hidden")
assert.Equal(t, map[string]bool(nil), cr.Votes)
}
func TestRest_AnonVote(t *testing.T) {
@@ -500,7 +472,6 @@ func TestRest_AnonVote(t *testing.T) {
req.Header.Add("X-JWT", anonToken)
resp, err := client.Do(req)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
return resp.StatusCode
}
@@ -530,21 +501,12 @@ func TestRest_AnonVote(t *testing.T) {
assert.Equal(t, 1, cr.Score)
assert.Equal(t, 1, cr.Vote)
assert.Equal(t, map[string]bool(nil), cr.Votes)
assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs)
}
type MockFS struct{}
func (fs *MockFS) ReadFile(path string) ([]byte, error) {
return []byte(fmt.Sprintf("template %s", path)), nil
}
func TestRest_Email(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.privRest.templates = &MockFS{}
// issue good token
claims := token.Claims{
Handshake: &token.Handshake{ID: "dev::good@example.com"},
@@ -616,7 +578,6 @@ func TestRest_EmailNotification(t *testing.T) {
mockDestination := &notify.MockDest{}
srv.privRest.notifyService = notify.NewService(srv.DataService, 1, mockDestination)
defer srv.privRest.notifyService.Close()
client := http.Client{}
@@ -632,14 +593,14 @@ func TestRest_EmailNotification(t *testing.T) {
assert.NoError(t, err)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
parentComment := store.Comment{}
require.NoError(t, render.DecodeJSON(strings.NewReader(string(body)), &parentComment))
// wait for mock notification Submit to kick off
time.Sleep(time.Millisecond * 30)
require.Equal(t, 1, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[0].Emails)
require.Equal(t, 2, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[0].Email)
assert.Equal(t, "admin@example.org", mockDestination.Get()[1].Email)
// create child comment from another user, email notification only to admin expected
req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(fmt.Sprintf(
@@ -649,17 +610,17 @@ func TestRest_EmailNotification(t *testing.T) {
"locator":{"url": "https://radio-t.com/blah1",
"site": "remark42"}}`, parentComment.ID)))
assert.NoError(t, err)
req.Header.Add("X-JWT", anonToken)
req.Header.Add("X-JWT", devToken)
resp, err = client.Do(req)
assert.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
// wait for mock notification Submit to kick off
time.Sleep(time.Millisecond * 30)
require.Equal(t, 2, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[1].Emails)
require.Equal(t, 4, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[2].Email)
assert.Equal(t, "admin@example.org", mockDestination.Get()[3].Email)
// send confirmation token for email
req, err = http.NewRequest(http.MethodPost, ts.URL+"/api/v1/email/subscribe?site=remark42&address=good@example.com", nil)
@@ -669,13 +630,12 @@ func TestRest_EmailNotification(t *testing.T) {
require.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
// wait for mock notification Submit to kick off
time.Sleep(time.Millisecond * 30)
require.Equal(t, 1, len(mockDestination.GetVerify()))
assert.Equal(t, "good@example.com", mockDestination.GetVerify()[0].Email)
verificationToken := mockDestination.GetVerify()[0].Token
require.Equal(t, 5, len(mockDestination.Get()))
require.NotEmpty(t, mockDestination.Get()[4].Verification)
verificationToken := mockDestination.Get()[4].Verification.Token
// verify email
req, err = http.NewRequest(http.MethodPost, ts.URL+fmt.Sprintf("/api/v1/email/confirm?site=remark42&tkn=%s", verificationToken), nil)
@@ -685,7 +645,6 @@ func TestRest_EmailNotification(t *testing.T) {
require.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
// get user information to verify the subscription
@@ -696,7 +655,6 @@ func TestRest_EmailNotification(t *testing.T) {
require.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
var user store.User
err = json.Unmarshal(body, &user)
@@ -712,17 +670,16 @@ func TestRest_EmailNotification(t *testing.T) {
"locator":{"url": "https://radio-t.com/blah1",
"site": "remark42"}}`, parentComment.ID)))
assert.NoError(t, err)
req.Header.Add("X-JWT", anonToken)
req.Header.Add("X-JWT", devToken)
resp, err = client.Do(req)
assert.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
// wait for mock notification Submit to kick off
time.Sleep(time.Millisecond * 30)
require.Equal(t, 3, len(mockDestination.Get()))
assert.Equal(t, []string{"good@example.com"}, mockDestination.Get()[2].Emails)
require.Equal(t, 7, len(mockDestination.Get()))
assert.Equal(t, "good@example.com", mockDestination.Get()[5].Email)
// delete user's email
req, err = http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/email?site=remark42", nil)
@@ -732,10 +689,9 @@ func TestRest_EmailNotification(t *testing.T) {
require.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))
// create child comment from another user, no email notification
// create child comment from another user, no email notification expected except for admin
req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", strings.NewReader(
`{"text": "test 321",
"user": {"name": "other_user"},
@@ -747,12 +703,11 @@ func TestRest_EmailNotification(t *testing.T) {
assert.NoError(t, err)
body, err = ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode, string(body))
// wait for mock notification Submit to kick off
time.Sleep(time.Millisecond * 30)
require.Equal(t, 4, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[3].Emails)
require.Equal(t, 9, len(mockDestination.Get()))
assert.Empty(t, mockDestination.Get()[7].Email)
}
func TestRest_UserAllData(t *testing.T) {
@@ -762,11 +717,11 @@ func TestRest_UserAllData(t *testing.T) {
// write 3 comments
user := store.User{ID: "dev", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
_, err := srv.DataService.Create(c1)
require.NoError(t, err, "%+v", err)
_, err = srv.DataService.Create(c2)
@@ -785,7 +740,6 @@ func TestRest_UserAllData(t *testing.T) {
ungzReader, err := gzip.NewReader(resp.Body)
assert.NoError(t, err)
require.NoError(t, resp.Body.Close())
ungzBody, err := ioutil.ReadAll(ungzReader)
assert.NoError(t, err)
strUungzBody := string(ungzBody)
@@ -808,7 +762,6 @@ func TestRest_UserAllData(t *testing.T) {
require.NoError(t, err)
resp, err = client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, 401, resp.StatusCode)
}
@@ -818,7 +771,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
user := store.User{ID: "dev", Name: "user name 1"}
c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
for i := 0; i < 51; i++ {
c.ID = fmt.Sprintf("id-%03d", i)
@@ -857,7 +810,6 @@ func TestRest_DeleteMe(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, resp.Body.Close())
assert.NoError(t, err)
m := map[string]string{}
@@ -866,18 +818,17 @@ func TestRest_DeleteMe(t *testing.T) {
assert.Equal(t, "remark42", m["site"])
assert.Equal(t, "dev", m["user_id"])
tkn := m["token"]
claims, err := srv.Authenticator.TokenService().Parse(tkn)
token := m["token"]
claims, err := srv.Authenticator.TokenService().Parse(token)
assert.NoError(t, err)
assert.Equal(t, "dev", claims.User.ID)
assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+tkn, m["link"])
assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"])
req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=remark42", ts.URL), nil)
assert.NoError(t, err)
resp, err = client.Do(req)
assert.NoError(t, err)
assert.Equal(t, 401, resp.StatusCode)
assert.NoError(t, resp.Body.Close())
}
func TestRest_SavePictureCtrl(t *testing.T) {
@@ -905,7 +856,6 @@ func TestRest_SavePictureCtrl(t *testing.T) {
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
m := map[string]string{}
err = json.Unmarshal(body, &m)
@@ -920,34 +870,29 @@ func TestRest_SavePictureCtrl(t *testing.T) {
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 1462, len(body))
assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
id = savePic("picture.gif")
resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
id = savePic("picture.jpg")
resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
id = savePic("picture.blah")
resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 400, resp.StatusCode)
}
@@ -966,7 +911,6 @@ func TestRest_CreateWithPictures(t *testing.T) {
EditDuration: 100 * time.Millisecond,
MaxSize: 2000,
})
defer imageService.Close(context.Background())
svc.privRest.imageService = imageService
svc.ImageService = imageService
+91 -9
View File
@@ -19,10 +19,10 @@ import (
R "github.com/go-pkgz/rest"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
type public struct {
@@ -31,6 +31,7 @@ type public struct {
readOnlyAge int
commentFormatter *store.CommentFormatter
imageService *image.Service
streamer *Streamer
webRoot string
}
@@ -158,6 +159,49 @@ func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
}
}
// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
sinceTs, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
return
}
fn := func() steamEventFn {
lastTS := sinceTs
lastCount := 0
return func() (event string, data []byte, upd bool, err error) {
key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
data, err = s.cache.Get(key, func() ([]byte, error) {
info, e := s.dataService.Info(locator, s.readOnlyAge)
if e != nil {
return nil, e
}
// cache update used as indication of post update. comparing lastTS for no-cache.
// removal won't update lastTS, count check will catch it.
if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
upd = true
}
lastTS = info.LastTS
lastCount = info.Count
return encodeJSONWithHTML(info)
})
if err != nil {
return "info", data, false, err
}
return "info", data, upd, nil
}
}
if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
}
}
// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
// limited with "since" param
func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
@@ -196,6 +240,45 @@ func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
}
}
// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get last comments stream for %s", siteID)
sinceTs, err := s.parseSince(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
return
}
if sinceTs.IsZero() {
sinceTs = time.Now()
}
fn := func() steamEventFn {
sinceTime := sinceTs
return func() (event string, data []byte, upd bool, err error) {
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
data, err = s.cache.Get(key, func() ([]byte, error) {
comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
if e != nil {
return nil, e
}
sinceTime = time.Now()
if len(comments) > 0 {
sinceTime = comments[0].Timestamp
upd = true
}
return encodeJSONWithHTML(comments)
})
return "last", data, upd, err
}
}
if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
}
}
// GET /id/{id}?site=siteID&url=post-url - gets a comment by id
func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
@@ -273,10 +356,9 @@ func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
// POST /counts?site=siteID - get number of comments for posts from post body
func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
const countBodyLimit int64 = 1024 * 128 // count request can be big for some site because it lists all urls
siteID := r.URL.Query().Get("site")
posts := []string{}
if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, countBodyLimit), &posts); err != nil {
if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
return
}
@@ -400,13 +482,13 @@ func (s *public) applyView(comments []store.Comment, view string) []store.Commen
}
func (s *public) parseSince(r *http.Request) (time.Time, error) {
sinceTS := time.Time{}
sinceTs := time.Time{}
if since := r.URL.Query().Get("since"); since != "" {
unixTS, e := strconv.ParseInt(since, 10, 64)
if e != nil {
return time.Time{}, errors.Wrap(e, "can't translate since parameter")
}
sinceTS = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
}
return sinceTS, nil
return sinceTs, nil
}
+343 -18
View File
@@ -1,11 +1,14 @@
package api
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -14,8 +17,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
func TestRest_Ping(t *testing.T) {
@@ -36,12 +39,10 @@ func TestRest_Preview(t *testing.T) {
assert.Equal(t, http.StatusOK, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, "<p>test 123</p>\n", string(b))
resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
assert.Equal(t, 400, resp.StatusCode)
}
@@ -210,9 +211,8 @@ func TestRest_FindReadOnly(t *testing.T) {
fmt.Sprintf("%s/api/v1/admin/readonly?site=remark42&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
assert.NoError(t, err)
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
_, err = client.Do(req)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
tree := service.Tree{}
res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1&format=tree")
@@ -445,7 +445,6 @@ func TestRest_Counts(t *testing.T) {
body, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.NoError(t, resp.Body.Close())
j := []store.PostInfo{}
err = json.Unmarshal(body, &j)
@@ -456,7 +455,6 @@ func TestRest_Counts(t *testing.T) {
resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
require.NoError(t, err)
assert.Equal(t, 400, resp.StatusCode)
assert.NoError(t, resp.Body.Close())
}
func TestRest_List(t *testing.T) {
@@ -528,15 +526,15 @@ func TestRest_Config(t *testing.T) {
j := R.JSON{}
err := json.Unmarshal([]byte(body), &j)
assert.NoError(t, err)
assert.Equal(t, 300.0, j["edit_duration"])
assert.Equal(t, 300., j["edit_duration"])
assert.EqualValues(t, []interface{}{"a1", "a2"}, j["admins"])
assert.Equal(t, "admin@remark-42.com", j["admin_email"])
assert.Equal(t, 4000.0, j["max_comment_size"])
assert.Equal(t, -5.0, j["low_score"])
assert.Equal(t, -10.0, j["critical_score"])
assert.Equal(t, 4000., j["max_comment_size"])
assert.Equal(t, -5., j["low_score"])
assert.Equal(t, -10., j["critical_score"])
assert.False(t, j["positive_score"].(bool))
assert.Equal(t, 10.0, j["readonly_age"])
assert.Equal(t, 10000.0, j["max_image_size"])
assert.Equal(t, 10., j["readonly_age"])
assert.Equal(t, 10000., j["max_image_size"])
assert.Equal(t, true, j["emoji_enabled"].(bool))
}
@@ -548,11 +546,11 @@ func TestRest_Info(t *testing.T) {
user := store.User{ID: "user1", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "remark42",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
_, err := srv.DataService.Create(c1)
require.NoError(t, err, "%+v", err)
@@ -568,7 +566,7 @@ func TestRest_Info(t *testing.T) {
err = json.Unmarshal([]byte(body), &info)
assert.NoError(t, err)
exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
FirstTS: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
assert.Equal(t, exp, info)
_, code = get(t, ts.URL+"/api/v1/info?site=remark42&url=https://radio-t.com/blah-no")
@@ -577,6 +575,145 @@ func TestRest_Info(t *testing.T) {
assert.Equal(t, 400, code)
}
func TestRest_InfoStream(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 1 * time.Millisecond
srv.pubRest.streamer.TimeOut = 800 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
postComment(t, ts.URL)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 10; i++ {
time.Sleep(10 * time.Millisecond)
postComment(t, ts.URL)
}
}()
body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
assert.Equal(t, 200, code)
<-done
recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
_, code = get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah123")
assert.Equal(t, 500, code)
}
func TestRest_InfoStreamTooMany(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 1 * time.Millisecond
srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
srv.pubRest.streamer.MaxActive = 10
postComment(t, ts.URL)
var errsCount int32
wg := sync.WaitGroup{}
wg.Add(20)
for i := 0; i < 20; i++ {
go func() {
_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
if code == 429 {
atomic.AddInt32(&errsCount, 1)
}
wg.Done()
}()
}
wg.Wait()
assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
}
func TestRest_InfoStreamTimeout(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
postComment(t, ts.URL)
st := time.Now()
_, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1")
assert.Equal(t, 200, code)
assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
}
func TestRest_InfoStreamCancel(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 1500 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
postComment(t, ts.URL)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 5; i++ {
time.Sleep(300 * time.Millisecond)
postComment(t, ts.URL)
}
}()
client := http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1", nil)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
defer cancel()
req = req.WithContext(ctx)
r, err := client.Do(req)
require.NoError(t, err)
defer r.Body.Close()
<-ctx.Done()
<-done
body, err := ioutil.ReadAll(r.Body)
require.EqualError(t, err, "context deadline exceeded")
assert.Equal(t, 200, r.StatusCode)
recs := strings.Count(string(body), "data:")
require.Equal(t, 1, recs, "should have 1 event:\n", string(body))
assert.Contains(t, string(body), `"count":2`)
}
func TestRest_InfoStreamSince(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 900 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
postComment(t, ts.URL)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 10; i++ {
time.Sleep(15 * time.Millisecond)
postComment(t, ts.URL)
}
}()
body, code := get(t, ts.URL+"/api/v1/stream/info?site=remark42&url=https://radio-t.com/blah1&since=12345678")
assert.Equal(t, 200, code)
<-done
recs := strings.Split(strings.TrimSuffix(body, "\n"), "\n")
require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
}
func TestRest_Robots(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()
@@ -588,3 +725,191 @@ func TestRest_Robots(t *testing.T) {
"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/user\nAllow: /api/v1/img\n"+
"Allow: /api/v1/avatar\nAllow: /api/v1/picture\n", body)
}
func TestRest_LastCommentsStream(t *testing.T) {
t.Skip() // TODO: enable after cache is migrated to https://github.com/dgraph-io/ristretto
ts, srv, teardown := startupT(t)
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 50 * time.Millisecond
srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
// stream endpoint currently relies on real cache being present
cacheBackend, err := cache.NewExpirableCache()
require.NoError(t, err)
memCache := cache.NewScache(cacheBackend)
srv.privRest.cache = memCache
srv.pubRest.cache = memCache
postComment(t, ts.URL)
defer teardown()
done := make(chan struct{})
go func() {
defer close(done)
for i := 1; i < 10; i++ {
postComment(t, ts.URL)
time.Sleep(100 * time.Millisecond)
}
t.Log("wrote 10 records")
}()
client := http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
require.NoError(t, err)
r, err := client.Do(req)
require.NoError(t, err)
defer r.Body.Close()
<-done
body, err := ioutil.ReadAll(r.Body)
require.NoError(t, err)
assert.Equal(t, 200, r.StatusCode)
assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
assert.Equal(t, "keep-alive", r.Header.Get("connection"))
recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
require.Equal(t, 9*3, len(recs), "9 events")
assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
}
func TestRest_LastCommentsStreamTimeout(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
postComment(t, ts.URL)
st := time.Now()
_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
assert.Equal(t, 200, code)
assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
}
func TestRest_LastCommentsStreamCancel(t *testing.T) {
ts, srv, teardown := startupT(t)
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
// stream endpoint currently relies on real cache being present
cacheBackend, err := cache.NewExpirableCache()
require.NoError(t, err)
memCache := cache.NewScache(cacheBackend)
srv.privRest.cache = memCache
srv.pubRest.cache = memCache
postComment(t, ts.URL)
defer teardown()
done := make(chan struct{})
go func() {
defer close(done)
for i := 1; i < 10; i++ {
time.Sleep(100 * time.Millisecond)
postComment(t, ts.URL)
}
}()
client := http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42", nil)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
defer cancel()
req = req.WithContext(ctx)
r, err := client.Do(req)
require.NoError(t, err)
<-done
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
require.EqualError(t, err, "context deadline exceeded")
assert.Equal(t, 200, r.StatusCode)
recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
assert.True(t, len(recs) < 30, "less 10 events")
}
func TestRest_LastCommentsStreamTooMany(t *testing.T) {
ts, srv, teardown := startupT(t)
defer teardown()
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 1 * time.Millisecond
srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
srv.pubRest.streamer.MaxActive = 10
postComment(t, ts.URL)
var errsCount int32
wg := sync.WaitGroup{}
wg.Add(20)
for i := 0; i < 20; i++ {
go func() {
_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
if code == 429 {
atomic.AddInt32(&errsCount, 1)
}
wg.Done()
}()
}
wg.Wait()
assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
_, code := get(t, ts.URL+"/api/v1/stream/last?site=remark42")
assert.Equal(t, 200, code, "all streams closed, good to go again")
}
func TestRest_LastCommentsStreamSince(t *testing.T) {
ts, srv, teardown := startupT(t)
srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
srv.pubRest.streamer.Refresh = 10 * time.Millisecond
srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
srv.pubRest.streamer.MaxActive = 100
// stream endpoint currently relies on real cache being present
cacheBackend, err := cache.NewExpirableCache()
require.NoError(t, err)
memCache := cache.NewScache(cacheBackend)
srv.privRest.cache = memCache
srv.pubRest.cache = memCache
postComment(t, ts.URL)
defer teardown()
done := make(chan struct{})
go func() {
defer close(done)
for i := 1; i < 10; i++ {
time.Sleep(50 * time.Millisecond)
postComment(t, ts.URL)
}
}()
client := http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=remark42&since=123456", nil)
require.NoError(t, err)
r, err := client.Do(req)
require.NoError(t, err)
<-done
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
require.NoError(t, err)
assert.Equal(t, 200, r.StatusCode)
assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
require.Equal(t, 10*3, len(recs), "should be 10 events, including first record:\n", recs)
}
func postComment(t *testing.T, url string) {
resp, err := post(t, url+"/api/v1/comment",
`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark42"}}`)
require.NoError(t, err)
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
}
+42 -83
View File
@@ -25,17 +25,16 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"go.uber.org/goleak"
"github.com/umputun/remark42/backend/app/migrator"
"github.com/umputun/remark42/backend/app/notify"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/rest/proxy"
"github.com/umputun/remark42/backend/app/store"
adminstore "github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark42/backend/app/store/service"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/notify"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/service"
)
var devToken = `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg`
@@ -100,11 +99,11 @@ func TestRest_Shutdown(t *testing.T) {
func TestRest_filterComments(t *testing.T) {
user := store.User{ID: "user1", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 10, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 20, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 5, 27, 1, 14, 25, 0, time.Local)}
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool {
return c.Text == "test test #1" || c.Text == "test test #3"
@@ -211,17 +210,14 @@ func TestRest_rejectAnonUser(t *testing.T) {
resp, err := http.Get(ts.URL)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
}
@@ -326,64 +322,28 @@ func TestRest_cacheControl(t *testing.T) {
assert.Equal(t, http.StatusOK, resp.StatusCode)
t.Logf("%+v", resp.Header)
assert.Equal(t, `"`+tt.etag+`"`, resp.Header.Get("Etag"))
assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds()))+", no-cache", resp.Header.Get("Cache-Control"))
assert.Equal(t, `max-age=`+strconv.Itoa(int(tt.exp.Seconds())), resp.Header.Get("Cache-Control"))
})
}
}
func TestRest_frameAncestors(t *testing.T) {
tbl := []struct {
hosts []string
header string
}{
{[]string{"http://example.com"}, "frame-ancestors http://example.com;"},
{[]string{}, ""},
{[]string{"http://example.com", "http://example2.com"}, "frame-ancestors http://example.com http://example2.com;"},
}
for i, tt := range tbl {
tt := tt
t.Run(strconv.Itoa(i), func(t *testing.T) {
req := httptest.NewRequest("GET", "http://example.com", nil)
w := httptest.NewRecorder()
h := frameAncestors(tt.hosts)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
h.ServeHTTP(w, req)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
t.Logf("%+v", resp.Header)
assert.Equal(t, tt.header, resp.Header.Get("Content-Security-Policy"))
})
}
}
// randomPath pick a file or folder name which is not in use for sure
func randomPath(tempDir, basename, suffix string) (string, error) {
for i := 0; i < 10; i++ {
fname := fmt.Sprintf("/%s/%s-%d%s", tempDir, basename, rand.Int31(), suffix)
fmt.Printf("fname %q", fname)
_, err := os.Stat(fname)
if err != nil {
return fname, nil
}
}
return "", errors.New("cannot create temp file")
}
func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
tmp := os.TempDir()
testDB, err := randomPath(tmp, "test-remark", ".db")
require.NoError(t, err)
var testDb string
// pick a file name which is not in use for sure
for i := 0; i < 10; i++ {
testDb = fmt.Sprintf("/%s/test-remark-%d.db", tmp, rand.Int31())
_, err := os.Stat(testDb)
if err != nil {
break
}
}
_ = os.RemoveAll(tmp + "/ava-remark42")
_ = os.RemoveAll(tmp + "/pics-remark42")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "remark42"})
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "remark42"})
require.NoError(t, err)
memCache := cache.NewScache(cache.NewNopCache())
@@ -407,9 +367,10 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
SecretReader: token.SecretFunc(func(aud string) (string, error) { return "secret", nil }),
AvatarStore: avatar.NewLocalFS(tmp + "/ava-remark42"),
}),
Cache: memCache,
WebRoot: tmp,
RemarkURL: "https://demo.remark42.com",
Cache: memCache,
WebRoot: tmp,
RemarkURL: "https://demo.remark42.com",
AdminEmail: "admin@example.org",
ImageService: image.NewService(&image.FileSystem{
Location: tmp + "/pics-remark42",
Partitions: 100,
@@ -430,6 +391,11 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
Cache: memCache,
KeyStore: astore,
},
Streamer: &Streamer{
Refresh: 100 * time.Millisecond,
TimeOut: 5 * time.Second,
MaxActive: 100,
},
NotifyService: notify.NopService,
EmojiEnabled: true,
}
@@ -440,7 +406,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
teardown = func() {
ts.Close()
require.NoError(t, srv.DataService.Close())
_ = os.Remove(testDB)
_ = os.Remove(testDb)
_ = os.RemoveAll(tmp + "/ava-remark42")
_ = os.RemoveAll(tmp + "/pics-remark42")
}
@@ -462,19 +428,19 @@ func fakeAuth(next http.Handler) http.Handler {
return http.HandlerFunc(fn)
}
func get(t *testing.T, url string) (response string, statusCode int) {
func get(t *testing.T, url string) (string, int) {
r, err := http.Get(url)
require.NoError(t, err)
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
require.NoError(t, err)
require.NoError(t, r.Body.Close())
return string(body), r.StatusCode
}
func sendReq(_ *testing.T, r *http.Request, tkn string) (*http.Response, error) {
func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
client := http.Client{Timeout: 5 * time.Second}
if tkn != "" {
r.Header.Set("X-JWT", tkn)
if token != "" {
r.Header.Set("X-JWT", token)
}
return client.Do(r)
}
@@ -486,25 +452,25 @@ func getWithDevAuth(t *testing.T, url string) (body string, code int) {
req.Header.Add("X-JWT", devToken)
r, err := client.Do(req)
require.NoError(t, err)
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
assert.NoError(t, err)
require.NoError(t, r.Body.Close())
return string(b), r.StatusCode
}
func getWithAdminAuth(t *testing.T, url string) (response string, statusCode int) {
func getWithAdminAuth(t *testing.T, url string) (string, int) {
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest("GET", url, nil)
require.NoError(t, err)
req.SetBasicAuth("admin", "password")
r, err := client.Do(req)
require.NoError(t, err)
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
assert.NoError(t, err)
require.NoError(t, r.Body.Close())
return string(body), r.StatusCode
}
func post(t *testing.T, url, body string) (*http.Response, error) {
func post(t *testing.T, url string, body string) (*http.Response, error) {
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest("POST", url, strings.NewReader(body))
assert.NoError(t, err)
@@ -524,7 +490,6 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
require.NoError(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
b, err = ioutil.ReadAll(resp.Body)
require.NoError(t, resp.Body.Close())
require.NoError(t, err)
crResp := R.JSON{}
@@ -537,12 +502,10 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
func requireAdminOnly(t *testing.T, req *http.Request) {
resp, err := sendReq(t, req, "") // no-auth user
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 401, resp.StatusCode)
resp, err = sendReq(t, req, devToken) // non-admin user
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, 403, resp.StatusCode)
}
@@ -568,7 +531,3 @@ func waitForHTTPSServerStart(port int) {
}
}
}
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
+2 -2
View File
@@ -10,8 +10,8 @@ import (
"github.com/gorilla/feeds"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
)
type rss struct {
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestServer_RssPost(t *testing.T) {
@@ -279,7 +279,7 @@ func waitOnSecChange() {
}
// clean formatting, i.e. multiple spaces, \t, \n
func cleanRssFormatting(expected, actual string) (cleanExp, cleanAct string) {
func cleanRssFormatting(expected, actual string) (string, string) {
reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
expected = strings.Replace(expected, "\n", " ", -1)
+103
View File
@@ -0,0 +1,103 @@
package api
import (
"context"
"fmt"
"io"
"net/http"
"sync/atomic"
"time"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
)
// Streamer creates endless stream of \n separated json records send to remote client
type Streamer struct {
TimeOut time.Duration
Refresh time.Duration
MaxActive int32
activeCount int32
}
type steamEventFn func() (event string, data []byte, upd bool, err error)
type steamEventResp struct {
data []byte
event string
err error
}
// Activate starts blocking function streaming update created by eventFn to ResponseWriter
// canceled on context or inactivity timeout
// note: eventFn is a closure needed to allow state management inside eventFn
func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
updCh := s.eventsCh(ctx, eventFn())
count := atomic.AddInt32(&s.activeCount, 1)
defer atomic.AddInt32(&s.activeCount, -1)
if count > s.MaxActive {
return errors.New("too many streams")
}
if ww, ok := w.(http.ResponseWriter); ok {
ww.Header().Set("Content-Type", "text/event-stream")
ww.Header().Set("Connection", "keep-alive")
ww.Header().Set("Cache-Control", "no-cache")
}
for {
select {
case <-ctx.Done(): // request closed by remote client
log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
return nil
case <-time.After(s.TimeOut): // request closed by timeout
log.Printf("[DEBUG] stream closed due to timeout")
return nil
case resp, ok := <-updCh: // new update
if !ok { // closed updCh
return nil
}
if resp.err != nil {
return resp.err
}
// make server-sent event record
// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
return errors.Wrap(e, "send to stream failed")
}
if fw, okFlush := w.(http.Flusher); okFlush {
fw.Flush()
}
}
}
}
// populate updates to chan, break on context close
func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
ch := make(chan steamEventResp)
go func() {
tick := time.NewTicker(s.Refresh)
defer func() {
close(ch)
tick.Stop()
}()
for {
select {
case <-ctx.Done(): // request closed by remote client
return
case <-tick.C:
event, resp, upd, err := fn()
if err != nil {
ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
return
}
if upd {
ch <- steamEventResp{event: event, data: resp, err: nil}
}
}
}
}()
return ch
}
+61
View File
@@ -0,0 +1,61 @@
package api
import (
"bytes"
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestStream_Timeout(t *testing.T) {
s := Streamer{
Refresh: 10 * time.Millisecond,
TimeOut: 100 * time.Millisecond,
MaxActive: 10,
}
eventFn := func() steamEventFn {
n := 0
return func() (event string, data []byte, upd bool, err error) {
n++
if n%2 == 0 || n > 10 {
return "test", nil, false, nil
}
return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
}
}
buf := bytes.Buffer{}
err := s.Activate(context.Background(), eventFn, &buf)
assert.NoError(t, err)
assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
}
func TestStream_Cancel(t *testing.T) {
s := Streamer{
Refresh: 10 * time.Millisecond,
TimeOut: 100 * time.Millisecond,
MaxActive: 10,
}
eventFn := func() steamEventFn {
n := 0
return func() (event string, data []byte, upd bool, err error) {
n++
if n%2 == 0 {
return "test", nil, false, nil
}
return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
}
}
buf := bytes.Buffer{}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
err := s.Activate(ctx, eventFn, &buf)
assert.NoError(t, err)
assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
}
+18 -12
View File
@@ -13,8 +13,6 @@ import (
"github.com/go-chi/render"
log "github.com/go-pkgz/lgr"
"github.com/go-pkgz/rest"
"github.com/umputun/remark42/backend/app/templates"
)
// All error codes for UI mapping and translation
@@ -40,6 +38,21 @@ const (
ErrAssetNotFound = 18 // requested file not found
)
const errorHTML = `<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">{{.Error}}: {{.Details}}.</p>
</div>
</body>
</html>
`
// errTmplData store data for error message
type errTmplData struct {
Error string
@@ -48,22 +61,15 @@ type errTmplData struct {
// SendErrorHTML makes html body with provided template and responds with provided http status code,
// error code is not included in render as it is intended for UI developers and not for the users
func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int, t templates.FileReader) {
func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
// MustExecute behaves like template.Execute, but panics if an error occurs.
MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) {
if err = tmpl.Execute(wr, data); err != nil {
panic(err)
}
}
MustRead := func(path string) string {
file, e := t.ReadFile(path)
if e != nil {
panic(e)
}
return string(file)
}
tmplstr := MustRead("error_response.html.tmpl")
tmpl := template.Must(template.New("error").Parse(tmplstr))
tmpl := template.Must(template.New("error").Parse(errorHTML))
log.Printf("[WARN] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
render.Status(r, httpStatusCode)
msg := bytes.Buffer{}
+3 -10
View File
@@ -2,7 +2,6 @@ package rest
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -11,7 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestSendErrorJSON(t *testing.T) {
@@ -38,18 +37,12 @@ func TestSendErrorJSON(t *testing.T) {
assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
}
type MockFS struct{}
func (fs *MockFS) ReadFile(path string) ([]byte, error) {
return []byte(fmt.Sprintf("{{.Error}}{{.Details}} %s", path)), nil
}
func TestSendErrorHTML(t *testing.T) {
fs := &MockFS{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/error" {
t.Log("http err request", r.URL)
SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987, fs)
SendErrorHTML(w, r, 500, errors.New("error 500"), "error details 123456", 987)
return
}
w.WriteHeader(404)
+2 -2
View File
@@ -15,8 +15,8 @@ import (
"github.com/go-pkgz/repeater"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/rest"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store/image"
)
// Image extracts image src from comment's html and provides proxy for them
+1 -1
View File
@@ -16,7 +16,7 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store/image"
)
// gopher png for test, from https://golang.org/src/image/png/example_test.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/go-pkgz/auth/token"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// MustGetUserInfo fails if can't extract user data from the request.
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestUser_GetUserInfo(t *testing.T) {
+6 -6
View File
@@ -10,7 +10,7 @@ import (
// Store defines interface returning admins info for given site
type Store interface {
Key(siteID string) (key string, err error)
Key() (key string, err error)
Admins(siteID string) (ids []string, err error)
Email(siteID string) (email string, err error)
Enabled(siteID string) (ok bool, err error)
@@ -37,9 +37,9 @@ type StaticStore struct {
}
// NewStaticStore makes StaticStore instance with given key
func NewStaticStore(key string, sites, adminIDs []string, email string) *StaticStore {
log.Printf("[DEBUG] admin users %+v, email %s", adminIDs, email)
return &StaticStore{key: key, sites: sites, admins: adminIDs, email: email}
func NewStaticStore(key string, sites []string, admins []string, email string) *StaticStore {
log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
return &StaticStore{key: key, sites: sites, admins: admins, email: email}
}
// NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
@@ -48,14 +48,14 @@ func NewStaticKeyStore(key string) *StaticStore {
}
// Key returns static key, same for all sites
func (s *StaticStore) Key(_ string) (key string, err error) {
func (s *StaticStore) Key() (key string, err error) {
if s.key == "" {
return "", errors.New("empty key for static key store")
}
return s.key, nil
}
// Admins returns static list of admin ids, the same for all sites
// Admins returns static list of admin's ids, the same for all sites
func (s *StaticStore) Admins(string) (ids []string, err error) {
return s.admins, nil
}
+2 -3
View File
@@ -7,10 +7,9 @@ import (
)
func TestStaticStore_Get(t *testing.T) {
var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"},
[]string{"123", "xyz"}, "aa@example.com")
var ks Store = NewStaticStore("key123", []string{"s1", "s2", "s3"}, []string{"123", "xyz"}, "aa@example.com")
k, err := ks.Key("any")
k, err := ks.Key()
assert.NoError(t, err, "valid store")
assert.Equal(t, "key123", k, "valid site")
+5 -5
View File
@@ -18,8 +18,8 @@ type RPC struct {
}
// Key returns the key, same for all sites
func (r *RPC) Key(siteID string) (key string, err error) {
resp, err := r.Call("admin.key", siteID)
func (r *RPC) Key() (key string, err error) {
resp, err := r.Call("admin.key")
if err != nil {
return "", err
}
@@ -35,7 +35,7 @@ func (r *RPC) Admins(siteID string) (ids []string, err error) {
return []string{}, err
}
if err := json.Unmarshal(*resp.Result, &ids); err != nil {
if err = json.Unmarshal(*resp.Result, &ids); err != nil {
return []string{}, err
}
return ids, nil
@@ -48,7 +48,7 @@ func (r *RPC) Email(siteID string) (email string, err error) {
return "", err
}
if err := json.Unmarshal(*resp.Result, &email); err != nil {
if err = json.Unmarshal(*resp.Result, &email); err != nil {
return "", err
}
return email, nil
@@ -61,7 +61,7 @@ func (r *RPC) Enabled(siteID string) (ok bool, err error) {
return false, err
}
if err := json.Unmarshal(*resp.Result, &ok); err != nil {
if err = json.Unmarshal(*resp.Result, &ok); err != nil {
return false, err
}
return ok, nil
+3 -3
View File
@@ -19,15 +19,15 @@ import (
)
func TestRemote_Key(t *testing.T) {
ts := testServer(t, `{"method":"admin.key","params":"any","id":1}`,
`{"result":"12345","params":"any","id":1}`)
ts := testServer(t, `{"method":"admin.key","id":1}`,
`{"result":"12345","id":1}`)
defer ts.Close()
c := RPC{Client: jrpc.Client{API: ts.URL, Client: http.Client{}}}
var a Store = &c
_ = a
res, err := c.Key("any")
res, err := c.Key()
assert.NoError(t, err)
assert.Equal(t, "12345", res)
t.Logf("%v %T", res, res)
-4
View File
@@ -27,7 +27,6 @@ type Comment struct {
Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
Pin bool `json:"pin,omitempty" bson:"pin,omitempty"`
Deleted bool `json:"delete,omitempty" bson:"delete"`
Imported bool `json:"imported,omitempty" bson:"imported"`
PostTitle string `json:"title,omitempty" bson:"title"`
}
@@ -84,7 +83,6 @@ func (c *Comment) PrepareUntrusted() {
c.ID = "" // don't allow user to define ID, force auto-gen
c.Timestamp = time.Time{} // reset time, force auto-gen
c.Votes = make(map[string]bool)
c.VotedIPs = make(map[string]VotedIPInfo)
c.Score = 0
c.Edit = nil
c.Pin = false
@@ -97,7 +95,6 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
c.Orig = ""
c.Score = 0
c.Votes = map[string]bool{}
c.VotedIPs = make(map[string]VotedIPInfo)
c.Edit = nil
c.Deleted = true
c.Pin = false
@@ -121,7 +118,6 @@ func (c *Comment) Sanitize() {
"|vi|vm|l|ld|s|sa|sb|sc|dl|sd|s2|se|sh|si|sx|sr|s1|ss|m|mb|mf|mh|mi|il" +
"|mo|o|ow|p|c|ch|cm|cp|cpf|c1|cs|g|gd|ge|gr|gh|gi|go|gp|gs|gu|gt|gl)$"
p.AllowAttrs("class").Matching(regexp.MustCompile(codeSpanClassRegex)).OnElements("span")
p.AllowAttrs("loading").Matching(regexp.MustCompile("^(lazy|eager)$")).OnElements("img")
c.Text = p.Sanitize(c.Text)
c.Orig = p.Sanitize(c.Orig)
c.User.ID = template.HTMLEscapeString(c.User.ID)
+1 -11
View File
@@ -14,7 +14,6 @@ func TestComment_Sanitize(t *testing.T) {
inp Comment
out Comment
}{
{inp: Comment{}, out: Comment{}},
{
inp: Comment{
@@ -67,12 +66,6 @@ func TestComment_Sanitize(t *testing.T) {
out: Comment{Text: "blah blah",
Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
},
{
inp: Comment{Text: "<scrİpt>&lt;img src=x onerror=alert(1)&gt;",
Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
out: Comment{Text: "&lt;img src=x onerror=alert(1)&gt;",
Locator: Locator{URL: "/p/2021/03/23/prep-747/#remark42__comment-1b365913-7056-4920-b9ad-01304bdda085"}},
},
}
for n, tt := range tbl {
@@ -104,7 +97,6 @@ func TestComment_PrepareUntrusted(t *testing.T) {
assert.Equal(t, time.Time{}, comment.Timestamp)
assert.Equal(t, false, comment.Deleted)
assert.Equal(t, make(map[string]bool), comment.Votes)
assert.Equal(t, make(map[string]VotedIPInfo), comment.VotedIPs)
assert.Equal(t, User{ID: "username"}, comment.User)
}
@@ -128,7 +120,6 @@ func TestComment_SetDeleted(t *testing.T) {
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
assert.Equal(t, map[string]bool{}, comment.Votes)
assert.Equal(t, map[string]VotedIPInfo{}, comment.VotedIPs)
assert.Equal(t, 0, comment.Score)
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
@@ -155,7 +146,6 @@ func TestComment_SetDeletedHard(t *testing.T) {
assert.Equal(t, "", comment.Text)
assert.Equal(t, "", comment.Orig)
assert.Equal(t, map[string]bool{}, comment.Votes)
assert.Equal(t, map[string]VotedIPInfo{}, comment.VotedIPs)
assert.Equal(t, 0, comment.Score)
assert.True(t, comment.Deleted)
assert.Nil(t, comment.Edit)
@@ -186,7 +176,7 @@ func TestComment_Snippet(t *testing.T) {
}
}
func TestComment_sanitizeAsURL(t *testing.T) {
func TestComment_SanitizeAsURL(t *testing.T) {
tbl := []struct {
inp, out string
+13 -17
View File
@@ -12,7 +12,7 @@ import (
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
@@ -55,7 +55,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
result := BoltDB{dbs: make(map[string]*bolt.DB)}
for _, site := range sites {
db, err := bolt.Open(site.FileName, 0600, &options) //nolint:gocritic //octalLiteral is OK as FileMode
db, err := bolt.Open(site.FileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
}
@@ -113,8 +113,8 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
// add reference to comment to "last" bucket
lastBkt = tx.Bucket([]byte(lastBucketName))
commentTS := []byte(comment.Timestamp.Format(tsNano))
if err = lastBkt.Put(commentTS, ref); err != nil {
commentTs := []byte(comment.Timestamp.Format(tsNano))
if err = lastBkt.Put(commentTs, ref); err != nil {
return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
}
@@ -123,7 +123,7 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
}
// put into individual user's bucket with ts as a key
if err = userBkt.Put(commentTS, ref); err != nil {
if err = userBkt.Put(commentTs, ref); err != nil {
return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
}
@@ -797,14 +797,6 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
if e = b.load(postBkt, commentID, &comment); e != nil {
return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
}
if !comment.Deleted {
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
}
// set deleted status and clear fields
comment.SetDeleted(mode)
@@ -818,6 +810,11 @@ func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID st
return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
}
// decrement comments count for post url
if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
}
return nil
})
}
@@ -847,7 +844,7 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
// deleteUser removes all comments and details for given user. Everything will be market as deleted
// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID, userID string, mode store.DeleteMode) error {
func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
// get list of all comments outside of transaction loop
posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
@@ -1011,8 +1008,7 @@ func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, er
}
info.Count++
info.LastTS = comment.Timestamp
err := b.save(infoBkt, comment.Locator.URL, &info)
return info, err
return info, b.save(infoBkt, comment.Locator.URL, &info)
}
func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
@@ -1028,7 +1024,7 @@ func (b *BoltDB) makeRef(comment store.Comment) []byte {
}
// parseRef gets parts of reference
func (b *BoltDB) parseRef(val []byte) (url, id string, err error) {
func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
elems := strings.Split(string(val), "!!")
if len(elems) != 2 {
return "", "", errors.Errorf("invalid reference value %s", string(val))
+12 -40
View File
@@ -10,10 +10,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
var testDB = "/tmp/test-remark.db"
var testDb = "/tmp/test-remark.db"
func TestBoltDB_CreateAndFind(t *testing.T) {
var b, teardown = prep(t)
@@ -221,13 +221,13 @@ func TestBoltDB_FindForUser(t *testing.T) {
}
func TestBoltDB_FindForUserPagination(t *testing.T) {
_ = os.Remove(testDB)
b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDB, SiteID: "radio-t"})
_ = os.Remove(testDb)
b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
require.NoError(t, err)
defer func() {
require.NoError(t, b.Close())
_ = os.Remove(testDB)
_ = os.Remove(testDb)
}()
c := store.Comment{
@@ -694,10 +694,6 @@ func TestBolt_DeleteComment(t *testing.T) {
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
// repeated deletion should not decrease comments count
err = b.Delete(delReq)
assert.NoError(t, err)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
@@ -811,22 +807,10 @@ func TestBoltAdmin_DeleteUserHard(t *testing.T) {
b, teardown := prep(t)
defer teardown()
comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
assert.NoError(t, err)
// soft delete one comment
delReq := DeleteRequest{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
CommentID: comments[0].ID,
DeleteMode: store.SoftDelete,
}
err = b.Delete(delReq)
assert.NoError(t, err)
err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
require.NoError(t, err)
comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
assert.NoError(t, err)
require.Equal(t, 2, len(comments), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
@@ -852,22 +836,10 @@ func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
b, teardown := prep(t)
defer teardown()
comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
assert.NoError(t, err)
// soft delete one comment
delReq := DeleteRequest{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
CommentID: comments[0].ID,
DeleteMode: store.SoftDelete,
}
err = b.Delete(delReq)
assert.NoError(t, err)
err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
require.NoError(t, err)
comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
assert.NoError(t, err)
require.Equal(t, 2, len(comments), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
@@ -921,9 +893,9 @@ func TestBoltDB_NewFailed(t *testing.T) {
// makes new boltdb, put two records
func prep(t *testing.T) (b *BoltDB, teardown func()) {
_ = os.Remove(testDB)
_ = os.Remove(testDb)
boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDB, SiteID: "radio-t"})
boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
assert.NoError(t, err)
b = boltStore
@@ -949,7 +921,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) {
teardown = func() {
require.NoError(t, b.Close())
_ = os.Remove(testDB)
_ = os.Remove(testDb)
}
return b, teardown
}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"strings"
"time"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
+2 -4
View File
@@ -2,10 +2,8 @@
package engine
import (
mock "github.com/stretchr/testify/mock"
store "github.com/umputun/remark42/backend/app/store"
)
import mock "github.com/stretchr/testify/mock"
import store "github.com/umputun/remark/backend/app/store"
// MockInterface is an autogenerated mock type for the Interface type
type MockInterface struct {
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestEngine_sortComments(t *testing.T) {
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"github.com/go-pkgz/jrpc"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// RPC implements remote engine and delegates all Calls to remote http server
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
func TestRemote_Create(t *testing.T) {
-17
View File
@@ -59,7 +59,6 @@ func (f *CommentFormatter) FormatText(txt string) (res string) {
res = conv.Convert(res)
}
res = f.shortenAutoLinks(res, shortURLLen)
res = f.lazyImage(res)
return res
}
@@ -109,19 +108,3 @@ func (f *CommentFormatter) unEscape(txt string) (res string) {
}
return res
}
// lazyImage adds loading=“lazy” attribute to all images
func (f *CommentFormatter) lazyImage(commentHTML string) (resHTML string) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
if err != nil {
return commentHTML
}
doc.Find("img").Each(func(i int, s *goquery.Selection) {
s.SetAttr("loading", "lazy")
})
resHTML, err = doc.Find("body").Html()
if err != nil {
return commentHTML
}
return resHTML
}
+1 -28
View File
@@ -1,7 +1,6 @@
package store
import (
"strconv"
"testing"
"time"
@@ -25,11 +24,6 @@ func TestFormatter_FormatText(t *testing.T) {
"<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0." +
"1/some-long-link/12345/6789012...</a></p>\n!converted", "links",
},
{
"something <img src=\"some.png\"/> _aaa_",
"<p>something <img src=\"some.png\" loading=\"lazy\"/> <em>aaa</em></p>\n!converted",
"lazy image",
},
{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
{"smth\n```go\nfunc main(aa string) int {return 0}\n```", `<p>smth</p>
<pre class="chroma"><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="nx">aa</span> <span class="kt">string</span><span class="p">)</span> <span class="kt">int</span> <span class="p">{</span><span class="k">return</span> <span class="mi">0</span><span class="p">}</span>
@@ -63,7 +57,7 @@ func TestFormatter_FormatComment(t *testing.T) {
User: User{ID: "username"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "http://example.com?foo=bar&x=123"},
Locator: Locator{SiteID: "site", URL: "url"},
Score: 10,
Pin: true,
Deleted: true,
@@ -120,24 +114,3 @@ func TestFormatter_ShortenAutoLinks(t *testing.T) {
assert.Equalf(t, tt.out, got, "check #%d", n)
}
}
func TestCommentFormatter_lazyImage(t *testing.T) {
tbl := []struct {
inp, out string
}{
{"", ""},
{`blah <img src="some.png" />`, `blah <img src="some.png" loading="lazy"/>`},
{`blah <img src="some.png" loading="lazy"/>`, `blah <img src="some.png" loading="lazy"/>`},
{`blah <img src="some.png"/> ххх <img src=http://example.com/pp2.jpg>`, `blah <img src="some.png" loading="lazy"/> ххх <img src="http://example.com/pp2.jpg" loading="lazy"/>`},
}
f := NewCommentFormatter(nil)
for i, tt := range tbl {
tt := tt
t.Run(strconv.Itoa(i), func(t *testing.T) {
assert.Equal(t, tt.out, f.lazyImage(tt.inp))
})
}
}
+1 -1
View File
@@ -25,7 +25,7 @@ type Bolt struct {
// NewBoltStorage create bolt image store
func NewBoltStorage(fileName string, options bolt.Options) (*Bolt, error) {
db, err := bolt.Open(fileName, 0600, &options) //nolint:gocritic //octalLiteral is OK as FileMode
db, err := bolt.Open(fileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
}
+5 -4
View File
@@ -8,9 +8,10 @@ import (
"testing"
"time"
bolt "go.etcd.io/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
)
func TestBoltStore_SaveCommit(t *testing.T) {
@@ -123,21 +124,21 @@ func TestBolt_Info(t *testing.T) {
assert.False(t, info.FirstStagingImageTS.IsZero())
}
func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket, id string) {
func assertBoltImgNil(t *testing.T, db *bolt.DB, bucket string, id string) {
checkBoltImgData(t, db, bucket, id, func(data []byte) error {
assert.Nil(t, data, id)
return nil
})
}
func assertBoltImgNotNil(t *testing.T, db *bolt.DB, bucket, id string) {
func assertBoltImgNotNil(t *testing.T, db *bolt.DB, bucket string, id string) {
checkBoltImgData(t, db, bucket, id, func(data []byte) error {
assert.NotNil(t, data, id)
return nil
})
}
func checkBoltImgData(t *testing.T, db *bolt.DB, bucket, id string, callback func([]byte) error) {
func checkBoltImgData(t *testing.T, db *bolt.DB, bucket string, id string, callback func([]byte) error) {
err := db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte(bucket))
assert.NotNil(t, bkt, "bucket %s not found", bucket)
+2 -2
View File
@@ -81,7 +81,7 @@ func (f *FileSystem) Load(id string) ([]byte, error) {
return nil, errors.Wrapf(err, "can't get image file for %s", id)
}
fh, err := os.Open(imgFile) //nolint:gosec // we open file from known location
fh, err := os.Open(imgFile) //nolint:gosec
if err != nil {
return nil, errors.Wrapf(err, "can't load image %s", id)
}
@@ -146,7 +146,7 @@ func (f *FileSystem) Info() (StoreInfo, error) {
// and avoid too many files in a single place.
// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
// Number of partitions defined by FileSystem.Partitions
func (f *FileSystem) location(base, id string) string {
func (f *FileSystem) location(base string, id string) string {
partition := func(id string) string {
f.crc.Do(func() {
+5 -20
View File
@@ -12,7 +12,6 @@ import (
"encoding/base64"
"fmt"
"image"
// support gif and jpeg images decoding
_ "image/gif"
_ "image/jpeg"
@@ -29,7 +28,6 @@ import (
"github.com/PuerkitoBio/goquery"
log "github.com/go-pkgz/lgr"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/rs/xid"
"golang.org/x/image/draw"
@@ -102,18 +100,6 @@ func NewService(s Store, p ServiceParams) *Service {
return &Service{ServiceParams: p, store: s}
}
// SubmitAndCommit multiple ids immediately
func (s *Service) SubmitAndCommit(idsFn func() []string) error {
errs := new(multierror.Error)
for _, id := range idsFn() {
err := s.store.Commit(id)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(err, "failed to commit image %s", id))
}
}
return errs.ErrorOrNil()
}
// Submit multiple ids via function for delayed commit
func (s *Service) Submit(idsFn func() []string) {
if idsFn == nil || s == nil {
@@ -131,11 +117,11 @@ func (s *Service) Submit(idsFn func() []string) {
for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.commitTTL {
time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
}
err := s.SubmitAndCommit(req.idsFn)
if err != nil {
log.Printf("[WARN] image commit error %v", err)
for _, id := range req.idsFn() {
if err := s.store.Commit(id); err != nil {
log.Printf("[WARN] failed to commit image %s", id)
}
}
atomic.AddInt32(&s.submitCount, -1)
}
log.Printf("[INFO] image submitter terminated")
@@ -143,7 +129,6 @@ func (s *Service) Submit(idsFn func() []string) {
})
atomic.AddInt32(&s.submitCount, 1)
s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
}
@@ -309,7 +294,7 @@ func resize(data []byte, limitW, limitH int) []byte {
}
// getProportionalSizes returns width and height resized by both dimensions proportionally
func getProportionalSizes(srcW, srcH, limitW, limitH int) (resW, resH int) {
func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
if srcW <= limitW && srcH <= limitH {
return srcW, srcH
+3 -6
View File
@@ -2,12 +2,9 @@
package image
import (
context "context"
time "time"
mock "github.com/stretchr/testify/mock"
)
import context "context"
import mock "github.com/stretchr/testify/mock"
import time "time"
// MockStore is an autogenerated mock type for the Store type
type MockStore struct {
+6 -8
View File
@@ -135,16 +135,14 @@ func TestService_Cleanup(t *testing.T) {
func TestService_Submit(t *testing.T) {
store := MockStore{}
store.On("Commit", mock.Anything, mock.Anything).Times(7).Return(nil)
svc := NewService(&store, ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100})
store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
svc := Service{store: &store, ServiceParams: ServiceParams{ImageAPI: "/blah/", EditDuration: time.Millisecond * 100}}
svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
err := svc.SubmitAndCommit(func() []string { return []string{"id4", "id5"} })
assert.NoError(t, err)
svc.Submit(func() []string { return []string{"id6", "id7"} })
svc.Submit(func() []string { return []string{"id4", "id5"} })
svc.Submit(nil)
store.AssertNumberOfCalls(t, "Commit", 2)
time.Sleep(time.Millisecond * 175)
store.AssertNumberOfCalls(t, "Commit", 7)
store.AssertNumberOfCalls(t, "Commit", 0)
time.Sleep(time.Millisecond * 150)
store.AssertNumberOfCalls(t, "Commit", 5)
svc.Close(context.TODO())
}
+3 -3
View File
@@ -29,7 +29,7 @@ func (r *RPC) Load(id string) ([]byte, error) {
return nil, err
}
var rawImg string
if err := json.Unmarshal(*resp.Result, &rawImg); err != nil {
if err = json.Unmarshal(*resp.Result, &rawImg); err != nil {
return nil, err
}
return ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(rawImg)))
@@ -54,8 +54,8 @@ func (r *RPC) Info() (StoreInfo, error) {
return StoreInfo{}, err
}
info := StoreInfo{}
if e := json.Unmarshal(*resp.Result, &info); e != nil {
return StoreInfo{}, e
if err = json.Unmarshal(*resp.Result, &info); err != nil {
return StoreInfo{}, err
}
return info, err
}
@@ -34,7 +34,7 @@ func NewRestrictedWordsMatcher(lister RestrictedWordsLister) *RestrictedWordsMat
}
// Match matches comment text against restricted words for specified site
func (m *RestrictedWordsMatcher) Match(siteID, text string) bool {
func (m *RestrictedWordsMatcher) Match(siteID string, text string) bool {
restrictedWords, err := m.lister.List(siteID)
if err != nil {
log.Printf("[WARN] failed to get restricted patterns for site %s: %v", siteID, err)
@@ -127,7 +127,7 @@ func (trie *wildcardTrie) addPattern(pattern string) {
// check tests if any pattern stored in trie matches the token. Recursive. Max depth is longest pattern in trie.
func (trie *wildcardTrie) check(token string) bool {
if token == "" {
if len(token) == 0 {
if trie.terminal {
return true
}
@@ -162,7 +162,7 @@ func (trie *wildcardTrie) check(token string) bool {
func (trie *wildcardTrie) checkAllSuffixes(token string) bool {
suffix := token
for {
if suffix == "" {
if len(suffix) == 0 {
return false
}
+51 -75
View File
@@ -9,16 +9,16 @@ import (
"sync"
"time"
"github.com/go-pkgz/lcw"
log "github.com/go-pkgz/lgr"
"github.com/google/uuid"
"github.com/hashicorp/go-multierror"
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
)
// DataStore wraps store.Interface with additional methods
@@ -45,7 +45,7 @@ type DataStore struct {
}
repliesCache struct {
lcw.LoadingCache
*cache.Cache
once sync.Once
}
}
@@ -101,36 +101,34 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
comment.PostTitle = title
}()
commentID, err = s.Engine.Create(comment)
s.submitImages(comment)
s.submitImages(comment.Locator, comment.ID)
if e := s.AdminStore.OnEvent(comment.Locator.SiteID, admin.EvCreate); e != nil {
log.Printf("[WARN] failed to send create event, %s", e)
}
return commentID, err
return s.Engine.Create(comment)
}
// Find wraps engine's Find call and alter results if needed. User used to alter comments
// in order to differentiate between user's comments vs others comments.
func (s *DataStore) Find(locator store.Locator, sortMethod string, user store.User) ([]store.Comment, error) {
return s.FindSince(locator, sortMethod, user, time.Time{})
func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
return s.FindSince(locator, sort, user, time.Time{})
}
// FindSince wraps engine's Find call and alter results if needed. Returns comments after since tx
func (s *DataStore) FindSince(locator store.Locator, sortMethod string, user store.User, since time.Time) ([]store.Comment, error) {
req := engine.FindRequest{Locator: locator, Sort: sortMethod, Since: since}
func (s *DataStore) FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error) {
req := engine.FindRequest{Locator: locator, Sort: sort, Since: since}
comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
changedSort := false
// sets votes controversy for comments added prior to #274
// also sanitizes locator.URL for comments added prior to #927
// set votes controversy for comments added prior to #274
for i, c := range comments {
if c.Controversy == 0 && len(c.Votes) > 0 {
c.Controversy = s.controversy(s.upsAndDowns(c))
if !changedSort && strings.Contains(sortMethod, "controversy") { // trigger sort change
if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
changedSort = true
}
}
@@ -139,7 +137,7 @@ func (s *DataStore) FindSince(locator store.Locator, sortMethod string, user sto
// resort commits if altered
if changedSort {
comments = engine.SortComments(comments, sortMethod)
comments = engine.SortComments(comments, sort)
}
return comments, nil
@@ -161,7 +159,7 @@ func (s *DataStore) Put(locator store.Locator, comment store.Comment) error {
}
// GetUserEmail gets user email
func (s *DataStore) GetUserEmail(siteID, userID string) (string, error) {
func (s *DataStore) GetUserEmail(siteID string, userID string) (string, error) {
res, err := s.Engine.UserDetail(engine.UserDetailRequest{
Detail: engine.UserEmail,
Locator: store.Locator{SiteID: siteID},
@@ -177,7 +175,7 @@ func (s *DataStore) GetUserEmail(siteID, userID string) (string, error) {
}
// SetUserEmail sets user email
func (s *DataStore) SetUserEmail(siteID, userID, value string) (string, error) {
func (s *DataStore) SetUserEmail(siteID string, userID string, value string) (string, error) {
res, err := s.Engine.UserDetail(engine.UserDetailRequest{
Detail: engine.UserEmail,
Locator: store.Locator{SiteID: siteID},
@@ -194,7 +192,7 @@ func (s *DataStore) SetUserEmail(siteID, userID, value string) (string, error) {
}
// DeleteUserDetail deletes user detail
func (s *DataStore) DeleteUserDetail(siteID, userID string, detail engine.UserDetail) error {
func (s *DataStore) DeleteUserDetail(siteID string, userID string, detail engine.UserDetail) error {
return s.Engine.Delete(engine.DeleteRequest{
Locator: store.Locator{SiteID: siteID},
UserID: userID,
@@ -219,42 +217,32 @@ func (s *DataStore) ResubmitStagingImages(sites []string) error {
comments, err := s.FindSince(locator, "time", store.User{}, ts)
result = multierror.Append(result, errors.Wrapf(err, "problem finding comments for site %s", site))
for _, c := range comments {
s.submitImages(c)
s.submitImages(c.Locator, c.ID)
}
}
return result.ErrorOrNil()
}
// submitImages initiated delayed commit of all images from the comment uploaded to remark42
func (s *DataStore) submitImages(comment store.Comment) {
idsFn := func() []string { // get all ids from comment's text
func (s *DataStore) submitImages(locator store.Locator, commentID string) {
s.ImageService.Submit(func() []string { // get all ids from comment's text
// this can be called after last edit, we have to retrieve fresh comment
cc, err := s.Engine.Get(engine.GetRequest{Locator: comment.Locator, CommentID: comment.ID})
cc, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
log.Printf("[WARN] can't get comment's %s text for image extraction, %v", comment.ID, err)
log.Printf("[WARN] can't get comment's %s text for image extraction, %v", commentID, err)
return nil
}
imgIds, err := s.ImageService.ExtractPictures(cc.Text)
if err != nil {
log.Printf("[WARN] can't get extract pictures from %s, %v", comment.ID, err)
log.Printf("[WARN] can't get extract pictures from %s, %v", commentID, err)
return nil
}
if len(imgIds) > 0 {
log.Printf("[DEBUG] image ids extracted from %s - %+v", comment.ID, imgIds)
log.Printf("[DEBUG] image ids extracted from %s - %+v", commentID, imgIds)
}
return imgIds
}
var err error
if comment.Imported {
err = s.ImageService.SubmitAndCommit(idsFn)
} else {
s.ImageService.Submit(idsFn)
}
if err != nil {
log.Printf("[WARN] failed to commit comment's images: %v", err)
}
})
}
// prepareNewComment sets new comment fields, hashing and sanitizing data
@@ -327,7 +315,7 @@ func (s *DataStore) Vote(req VoteReq) (comment store.Comment, err error) {
}
v, voted := comment.Votes[req.UserID]
if voted && v == req.Val { // voted before and same vote (+/-) again. Change allowed, i.e. +, - or -, + is fine
if voted && v == req.Val {
return comment, errors.Errorf("user %s already voted for %s", req.UserID, req.CommentID)
}
@@ -353,16 +341,9 @@ func (s *DataStore) Vote(req VoteReq) (comment store.Comment, err error) {
return comment, errors.Errorf("minimal score reached for comment %s", req.CommentID)
}
// add ip hash to voted ip map
if comment.VotedIPs == nil {
comment.VotedIPs = map[string]store.VotedIPInfo{}
}
comment.VotedIPs[userIPHash] = store.VotedIPInfo{Timestamp: time.Now(), Value: req.Val}
// reset vote if user changed to opposite. Effectively it is "forget about prev votes" to allow "+ - -" or "- + +" corrections
// reset vote if user changed to opposite
if voted && v != req.Val {
delete(comment.Votes, req.UserID)
delete(comment.VotedIPs, userIPHash)
}
// add to voted map if first vote
@@ -370,6 +351,13 @@ func (s *DataStore) Vote(req VoteReq) (comment store.Comment, err error) {
comment.Votes[req.UserID] = req.Val
}
// add ip hash to voted ip map
if comment.VotedIPs == nil {
comment.VotedIPs = map[string]store.VotedIPInfo{}
}
comment.VotedIPs[userIPHash] = store.VotedIPInfo{Timestamp: time.Now(), Value: req.Val}
// update score
if req.Val {
comment.Score++
@@ -487,11 +475,11 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
func (s *DataStore) HasReplies(comment store.Comment) bool {
s.repliesCache.once.Do(func() {
// default expiration time of 5 minutes and cleanup time of 2.5 minutes
s.repliesCache.LoadingCache, _ = lcw.NewExpirableCache(lcw.TTL(5 * time.Minute))
// default expiration time of 5 minutes, purge every 10 minutes
s.repliesCache.Cache = cache.New(5*time.Minute, 10*time.Minute)
})
if _, found := s.repliesCache.Peek(comment.ID); found {
if _, found := s.repliesCache.Get(comment.ID); found {
return true
}
@@ -505,10 +493,7 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
for _, c := range comments {
if c.ParentID != "" && !c.Deleted {
if c.ParentID == comment.ID {
// When this code is reached, key "comment.ID" is not in cache.
// Calling cache.Get on it will put it in cache with 5 minutes TTL.
// We call it with empty struct as value as we care about keys and not values.
_, _ = s.repliesCache.Get(comment.ID, func() (interface{}, error) { return struct{}{}, nil })
s.repliesCache.Set(comment.ID, true, cache.DefaultExpiration)
return true
}
}
@@ -605,7 +590,7 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
}
// IsAdmin checks if usesID in the list of admins
func (s *DataStore) IsAdmin(siteID, userID string) bool {
func (s *DataStore) IsAdmin(siteID string, userID string) bool {
admins, err := s.AdminStore.Admins(siteID)
if err != nil {
log.Printf("[WARN] can't get admins for %s, %v", siteID, err)
@@ -639,14 +624,14 @@ func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error {
}
// IsVerified checks if user verified
func (s *DataStore) IsVerified(siteID, userID string) bool {
func (s *DataStore) IsVerified(siteID string, userID string) bool {
req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified}
ro, err := s.Engine.Flag(req)
return err == nil && ro
}
// SetVerified set/reset verified status for user
func (s *DataStore) SetVerified(siteID, userID string, status bool) error {
func (s *DataStore) SetVerified(siteID string, userID string, status bool) error {
roStatus := engine.FlagFalse
if status {
roStatus = engine.FlagTrue
@@ -657,14 +642,14 @@ func (s *DataStore) SetVerified(siteID, userID string, status bool) error {
}
// IsBlocked checks if user blocked
func (s *DataStore) IsBlocked(siteID, userID string) bool {
func (s *DataStore) IsBlocked(siteID string, userID string) bool {
req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Blocked}
ro, err := s.Engine.Flag(req)
return err == nil && ro
}
// SetBlock set/reset verified status for user
func (s *DataStore) SetBlock(siteID, userID string, status bool, ttl time.Duration) error {
func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
roStatus := engine.FlagFalse
if status {
roStatus = engine.FlagTrue
@@ -710,13 +695,13 @@ func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.D
}
// DeleteUser removes all comments from user
func (s *DataStore) DeleteUser(siteID, userID string, mode store.DeleteMode) error {
func (s *DataStore) DeleteUser(siteID string, userID string, mode store.DeleteMode) error {
req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: mode}
return s.Engine.Delete(req)
}
// List of commented posts
func (s *DataStore) List(siteID string, limit, skip int) ([]store.PostInfo, error) {
func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
req := engine.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip}
return s.Engine.Info(req)
}
@@ -858,15 +843,7 @@ func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.U
// Close store service
func (s *DataStore) Close() error {
errs := new(multierror.Error)
if s.repliesCache.LoadingCache != nil {
errs = multierror.Append(errs, s.repliesCache.LoadingCache.Close())
}
if s.TitleExtractor != nil {
errs = multierror.Append(errs, s.TitleExtractor.Close())
}
errs = multierror.Append(errs, s.Engine.Close())
return errs.ErrorOrNil()
return s.Engine.Close()
}
func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
@@ -942,8 +919,7 @@ func (s *DataStore) prepVotes(c store.Comment, user store.User) store.Comment {
}
}
c.Votes = nil // hide voters list
c.VotedIPs = nil // hide voted ips (hashes)
c.Votes = nil // hide voters list
return c
}
@@ -951,7 +927,7 @@ func (s *DataStore) prepVotes(c store.Comment, user store.User) store.Comment {
// Note: secret shared across sites, but some sites can be disabled.
func (s *DataStore) getSecret(siteID string) (secret string, err error) {
if secret, err = s.AdminStore.Key("any"); err != nil {
if secret, err = s.AdminStore.Key(); err != nil {
return "", errors.Wrapf(err, "can't get secret for site %s", siteID)
}
+25 -68
View File
@@ -22,10 +22,10 @@ import (
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark42/backend/app/store/admin"
"github.com/umputun/remark42/backend/app/store/engine"
"github.com/umputun/remark42/backend/app/store/image"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/image"
)
func TestService_CreateFromEmpty(t *testing.T) {
@@ -52,7 +52,6 @@ func TestService_CreateFromEmpty(t *testing.T) {
assert.Equal(t, "name", res.User.Name)
assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
assert.Equal(t, map[string]bool(nil), res.Votes)
assert.Equal(t, map[string]store.VotedIPInfo(nil), res.VotedIPs)
}
func TestService_CreateSiteDisabled(t *testing.T) {
@@ -80,10 +79,8 @@ func TestService_CreateFromPartial(t *testing.T) {
Text: "text",
Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
Votes: map[string]bool{"u1": true, "u2": false},
VotedIPs: map[string]store.VotedIPInfo{"xxx": {Value: true, Timestamp: time.Now()},
"yyy": {Value: false, Timestamp: time.Now()}},
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
}
id, err := b.Create(comment)
assert.NoError(t, err)
@@ -107,7 +104,6 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
defer teardown()
b := DataStore{Engine: eng, AdminStore: ks,
TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
defer b.Close()
postPath := "/post/42"
postTitle := "Post Title 42"
@@ -172,7 +168,6 @@ func TestService_SetTitle(t *testing.T) {
defer teardown()
b := DataStore{Engine: eng, AdminStore: ks,
TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
defer b.Close()
comment := store.Comment{
Text: "text",
Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
@@ -197,9 +192,8 @@ func TestService_SetTitle(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "post1 blah 123", c.PostTitle)
bErr := DataStore{Engine: eng, AdminStore: ks}
defer bErr.Close()
_, err = bErr.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
b = DataStore{Engine: eng, AdminStore: ks}
_, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
require.EqualError(t, err, "no title extractor")
}
@@ -244,14 +238,12 @@ func TestService_Vote(t *testing.T) {
assert.Equal(t, 1, c.Score)
assert.Equal(t, 1, c.Vote, "can see own vote result")
assert.Nil(t, c.Votes)
assert.Nil(t, c.VotedIPs)
// check result as user2
c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user2"})
assert.NoError(t, err)
assert.Equal(t, 1, c.Score)
assert.Equal(t, 0, c.Vote, "can't see other user vote result")
assert.Nil(t, c.Votes)
assert.Nil(t, c.VotedIPs)
req = VoteReq{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -307,7 +299,6 @@ func TestService_Vote(t *testing.T) {
assert.Equal(t, 0, res[0].Score)
assert.Equal(t, 0, res[0].Vote)
assert.Equal(t, map[string]bool(nil), res[0].Votes, "vote reset ok")
assert.Equal(t, map[string]store.VotedIPInfo(nil), res[0].VotedIPs, "vote reset ok")
}
func TestService_VoteLimit(t *testing.T) {
@@ -390,11 +381,10 @@ func TestService_VoteAggressive(t *testing.T) {
assert.Equal(t, 2, res[0].Score, "add single +1")
assert.Equal(t, 1, res[0].Vote, "user1 voted +1")
assert.Equal(t, 0, len(res[0].Votes), "votes hidden")
assert.Equal(t, 0, len(res[0].VotedIPs), "vote ips hidden")
// random +1/-1 result should be [0..2]
rand.Seed(time.Now().UnixNano())
for i := 0; i < 100; i++ {
for i := 0; i < 1000; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -443,7 +433,6 @@ func TestService_VoteConcurrent(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 100, res[0].Score, "should have 100 score")
assert.Equal(t, 0, len(res[0].Votes), "should hide votes")
assert.Equal(t, 0, len(res[0].VotedIPs), "should hide vote ips")
assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
}
@@ -463,29 +452,7 @@ func TestService_VotePositive(t *testing.T) {
assert.NoError(t, err, "minimal score doesn't affect positive vote")
assert.Equal(t, 1, c.Score)
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-1",
UserID: "user2", Val: true})
assert.NoError(t, err, "vote set to +2")
assert.Equal(t, 2, c.Score)
// check +, -, -
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-1",
UserID: "user5", Val: true})
assert.NoError(t, err, "user5 +1, score 3")
assert.Equal(t, 3, c.Score)
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-1",
UserID: "user5", Val: false})
assert.NoError(t, err, "user5 -1, score reset to 2")
assert.Equal(t, 2, c.Score)
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-1",
UserID: "user5", Val: false})
assert.NoError(t, err, "user5 -1, score 1")
assert.Equal(t, 1, c.Score)
// allow negative voting
b.PositiveScore = false
b.PositiveScore = false // allow negative voting
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-1",
UserID: "user2", Val: false})
assert.NoError(t, err, "minimal score ignored")
@@ -534,7 +501,8 @@ func TestService_VoteSameIP(t *testing.T) {
eng, teardown := prepStoreEngine(t)
defer teardown()
b := DataStore{Engine: eng, AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
b := DataStore{Engine: eng, AdminStore: admin.NewStaticKeyStore("secret 123"),
MaxVotes: -1}
b.RestrictSameIPVotes.Enabled = true
c, err := b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-2",
@@ -545,17 +513,12 @@ func TestService_VoteSameIP(t *testing.T) {
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-2",
UserID: "user3", UserIP: "123", Val: true})
assert.EqualError(t, err, "the same ip cce61be6e0a692420ae0de31dceca179123c3b8a already voted for id-2")
assert.Equal(t, 1, c.Score, "still have 1 score, rejected")
assert.Equal(t, 1, c.Score, "still have 1 score")
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-2",
UserID: "user2", UserIP: "123", Val: false})
UserID: "user3", UserIP: "123", Val: false})
assert.NoError(t, err)
assert.Equal(t, 0, c.Score, "reset to 0 score, opposite vote allowed")
c, err = b.Vote(VoteReq{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, CommentID: "id-2",
UserID: "user2", UserIP: "123", Val: false})
assert.NoError(t, err)
assert.Equal(t, -1, c.Score, "set to -1 score, correction vote allowed")
}
func TestService_VoteSameIPWithDuration(t *testing.T) {
@@ -646,7 +609,6 @@ func TestService_EditComment(t *testing.T) {
eng, teardown := prepStoreEngine(t)
defer teardown()
b := DataStore{Engine: eng, AdminStore: admin.NewStaticKeyStore("secret 123")}
defer b.Close()
res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
@@ -676,7 +638,6 @@ func TestService_DeleteComment(t *testing.T) {
eng, teardown := prepStoreEngine(t)
defer teardown()
b := DataStore{Engine: eng, AdminStore: admin.NewStaticKeyStore("secret 123")}
defer b.Close()
res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
@@ -718,7 +679,6 @@ func TestService_EditCommentReplyFailed(t *testing.T) {
eng, teardown := prepStoreEngine(t)
defer teardown()
b := DataStore{Engine: eng, AdminStore: admin.NewStaticKeyStore("secret 123")}
defer b.Close()
res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[1])
@@ -922,6 +882,7 @@ func TestService_IsAdmin(t *testing.T) {
assert.False(t, b.IsAdmin("radio-t", "user1"))
assert.True(t, b.IsAdmin("radio-t", "user2"))
assert.False(t, b.IsAdmin("radio-t-bad", "user1"))
}
func TestService_HasReplies(t *testing.T) {
@@ -931,7 +892,6 @@ func TestService_HasReplies(t *testing.T) {
defer teardown()
b := DataStore{Engine: eng, EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticStore("secret 123", []string{"radio-t"}, []string{"user2"}, "user@email.com")}
defer b.Close()
comment := store.Comment{
ID: "id-1",
@@ -1343,7 +1303,7 @@ func TestService_submitImages(t *testing.T) {
_, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
assert.NoError(t, err)
b.submitImages(c)
b.submitImages(c.Locator, c.ID)
time.Sleep(250 * time.Millisecond)
mockStore.AssertNumberOfCalls(t, "Commit", 2)
}
@@ -1457,11 +1417,9 @@ func TestService_alterComment(t *testing.T) {
engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
svc := DataStore{Engine: &engineMock}
r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"},
Locator: store.Locator{URL: "http://example.com?foo=bar"}},
r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
store.User{Name: "dev", ID: "devid", Admin: false})
assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid"},
Locator: store.Locator{URL: "http://example.com?foo=bar"}}, r, "ip cleaned")
assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid"}}, r, "ip cleaned")
r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
store.User{Name: "dev", ID: "devid", Admin: true})
assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, r, "ip not cleaned")
@@ -1478,8 +1436,7 @@ func TestService_alterComment(t *testing.T) {
engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(true, nil)
engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
svc = DataStore{Engine: &engineMock}
r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true},
Locator: store.Locator{URL: "javascript:alert('XSS1')"}},
r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}},
store.User{Name: "dev", ID: "devid", Admin: false})
assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true, ID: "devid"},
Deleted: false}, r, "blocked")
@@ -1508,14 +1465,14 @@ func Benchmark_ServiceCreate(b *testing.B) {
}
// makes new boltdb, put two records
func prepStoreEngine(t *testing.T) (e engine.Interface, teardown func()) {
testDBLoc, err := ioutil.TempDir("", "test_image_r42")
func prepStoreEngine(t *testing.T) (engine.Interface, func()) {
testDbLoc, err := ioutil.TempDir("", "test_image_r42")
require.NoError(t, err)
testDB := path.Join(testDBLoc, "test.db")
_ = os.Remove(testDB)
testDb := path.Join(testDbLoc, "test.db")
_ = os.Remove(testDb)
st := time.Now()
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDB, SiteID: "radio-t"})
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
assert.NoError(t, err)
comment := store.Comment{
@@ -1540,7 +1497,7 @@ func prepStoreEngine(t *testing.T) (e engine.Interface, teardown func()) {
t.Logf("prepared store engine in %v", time.Since(st))
return boltStore, func() {
assert.NoError(t, boltStore.Close())
_ = os.Remove(testDB)
_ = os.Remove(testDb)
}
}
+2 -7
View File
@@ -40,7 +40,7 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {
// Get page for url and return title
func (t *TitleExtractor) Get(url string) (string, error) {
client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
b, err := t.cache.Get(url, func() (interface{}, error) {
b, err := t.cache.Get(url, func() (lcw.Value, error) {
resp, err := client.Get(url)
if err != nil {
return nil, errors.Wrapf(err, "failed to load page %s", url)
@@ -63,18 +63,13 @@ func (t *TitleExtractor) Get(url string) (string, error) {
// on error save result (empty string) to cache too and return "" title
if err != nil {
_, _ = t.cache.Get(url, func() (interface{}, error) { return "", nil })
_, _ = t.cache.Get(url, func() (lcw.Value, error) { return "", nil })
return "", err
}
return b.(string), nil
}
// Close title extractor
func (t *TitleExtractor) Close() error {
return t.cache.Close()
}
// get title from body reader, traverse recursively
func (t *TitleExtractor) getTitle(r io.Reader) (string, bool) {
doc, err := html.Parse(r)
-4
View File
@@ -30,7 +30,6 @@ func TestTitle_GetTitle(t *testing.T) {
}
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
defer ex.Close()
for i, tt := range tbl {
tt := tt
t.Run(fmt.Sprintf("check-%d", i), func(t *testing.T) {
@@ -43,7 +42,6 @@ func TestTitle_GetTitle(t *testing.T) {
func TestTitle_Get(t *testing.T) {
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
defer ex.Close()
var hits int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.String() == "/good" {
@@ -77,7 +75,6 @@ func TestTitle_GetConcurrent(t *testing.T) {
body += "something something blah blah\n"
}
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
defer ex.Close()
var hits int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.String(), "/good") {
@@ -106,7 +103,6 @@ func TestTitle_GetConcurrent(t *testing.T) {
func TestTitle_GetFailed(t *testing.T) {
ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
defer ex.Close()
var hits int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&hits, 1)
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"strings"
"time"
"github.com/umputun/remark42/backend/app/store"
"github.com/umputun/remark/backend/app/store"
)
// Tree is formatter making tree from the list of comments
@@ -79,7 +79,7 @@ func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree
}
// proc makes tree for one top-level comment recursively
func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (result *Node, modified, created time.Time) {
func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (*Node, time.Time, time.Time) {
if rd.tsModified.IsZero() || rd.tsCreated.IsZero() {
rd.tsModified, rd.tsCreated = node.Comment.Timestamp, node.Comment.Timestamp

Some files were not shown because too many files have changed in this diff Show More