mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into user-info-redux

Browse Source
This commit is contained in:
Igor Adamenko
2018-09-29 22:50:33 +03:00
committed by GitHub
parent d09e45b0a8 72aeadcfd1
commit a6e2fd30c4
255 changed files with 30505 additions and 2092 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+3 -1
View File
@@ -5,7 +5,7 @@
/web/public/
/.vscode/
/.idea/
/.git/objects/
/.git/
# source files
docker-compose.yml
@@ -19,3 +19,5 @@ debug
debug.test
*.prof
*.test
/bin/
remark42
.drone.yml
+21 -1
View File
@@ -7,13 +7,16 @@ pipeline:
build:
image: golang:1.10-alpine
commands:
- sleep 5
- nslookup mongo
- nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
- cd backend/app
- go build -v ./...
docker_master:
image: plugins/docker
repo: umputun/remark42
secrets: [ docker_username, docker_password ]
secrets: [ docker_username, docker_password]
build_args:
- DRONE=${DRONE}
- DRONE_TAG=${DRONE_TAG}
@@ -39,6 +42,17 @@ pipeline:
when:
event: tag
artifacts_tag:
image: plugins/docker
dockerfile: Dockerfile.artifacts
build_args:
- DRONE=${DRONE}
- DRONE_TAG=${DRONE_TAG}
- DRONE_COMMIT=${DRONE_COMMIT}
- GITHUB_TOKEN=${GITHUB_TOKEN}
when:
event: tag
docker_branch:
image: plugins/docker
repo: umputun/remark42
@@ -79,3 +93,9 @@ pipeline:
secrets: [ email_username, email_password ]
when:
status: [ changed, failure ]
services:
mongo:
image: mongo:3.6
command: [ --smallfiles ]
.gitignore
+3
View File
@@ -15,3 +15,6 @@ debug.test
*.test
/rest-client.env.json
.DS_Store
.mongo
remark42
/bin/
.travis.yml
+6
View File
@@ -3,6 +3,9 @@ install:
- docker-compose --version
script:
- docker run -d --name=mongo mongo:3.6 && sleep 3
- export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
- echo "running mongo on $MONGO_TEST"
- docker build
--build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
--build-arg CI=$CI
@@ -16,4 +19,7 @@ script:
--build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
--build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
--build-arg TRAVIS_TAG=$TRAVIS_TAG
--build-arg MONGO_TEST=$MONGO_TEST
.
- docker rm -f mongo
Dockerfile
+34 -18
View File
@@ -19,34 +19,46 @@ ARG DRONE_BRANCH
ARG DRONE_PULL_REQUEST
ARG SKIP_BACKEND_TEST
ARG MONGO_TEST
WORKDIR /go/src/github.com/umputun/remark/backend
ADD backend /go/src/github.com/umputun/remark/backend
RUN cd app && \
# run tests
RUN \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
else echo "skip backend test" ; fi
RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
# linters
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
--enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
--enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
--enable=deadcode --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \
else echo "skip backend linters" ; fi
# coverage test, submit to coverals if COVERALLS_TOKEN in env
RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
echo coverall not enabled ; \
else \
mkdir -p target && /script/coverage.sh && \
goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
# coverage report
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
mkdir -p target && /script/coverage.sh ; \
else echo "skip backend coverage" ; fi
# get revision from git. if DRONE presented use DRONE_* git env to make version
# submit coverage to coverals if COVERALLS_TOKEN in env
RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
echo "coverall not enabled" ; \
else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
# if DRONE presented use DRONE_* git env to make version
RUN \
if [ -z "$DRONE" ] ; then \
echo "runs outside of drone" && version="local"; \
else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
echo "version=$version" && \
go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app
go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
FROM node:10.6-alpine as build-frontend-deps
@@ -56,15 +68,16 @@ ARG CI
RUN apk add --no-cache --update git
ADD web/package.json /srv/web/package.json
ADD web/package-lock.json /srv/web/package-lock.json
RUN cd /srv/web && npm ci
RUN cd /srv/web && CI=true npm ci
FROM node:10.6-alpine as build-frontend
ARG CI
ARG SKIP_FRONTEND_TEST
ARG NODE_ENV=production
ADD web /srv/web
COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
ADD web /srv/web
RUN cd /srv/web && \
if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
else echo "skip frontend tests and lint" ; npm run build ; fi && \
@@ -75,16 +88,19 @@ FROM umputun/baseimage:app-latest
WORKDIR /srv
ADD backend/scripts/*.sh /srv/
ADD start.sh /srv/start.sh
RUN chmod +x /srv/*.sh
ADD entrypoint.sh /entrypoint.sh
ADD backend/scripts/backup.sh /usr/local/bin/backup
ADD backend/scripts/restore.sh /usr/local/bin/restore
ADD backend/scripts/import.sh /usr/local/bin/import
RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark /srv/
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
COPY --from=build-frontend /srv/web/public/ /srv/web
RUN chown -R app:app /srv
RUN ln -s /srv/remark42 /usr/bin/remark42
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
CMD ["/srv/start.sh"]
ENTRYPOINT ["/init.sh"]
CMD ["server"]
ENTRYPOINT ["/entrypoint.sh"]
Dockerfile.artifacts
+91
View File
@@ -0,0 +1,91 @@
FROM node:10.6-alpine as build-frontend-deps
ARG CI
ENV SKIP_FRONTEND_TEST=true
RUN apk add --no-cache --update git
ADD web/package.json /srv/web/package.json
ADD web/package-lock.json /srv/web/package-lock.json
RUN cd /srv/web && CI=true npm ci
FROM node:10.6-alpine as build-frontend
ARG CI
ARG NODE_ENV=production
ENV SKIP_FRONTEND_TEST=true
COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
ADD web /srv/web
RUN cd /srv/web && \
npm run build && \
rm -rf ./node_modules
FROM umputun/baseimage:buildgo-latest as build-backend
ARG GITHUB_TOKEN
ENV SKIP_BACKEND_TEST=true
WORKDIR /go/src/github.com/umputun/remark/backend
ADD backend /go/src/github.com/umputun/remark/backend
ADD README.md /go/src/github.com/umputun/remark/
ADD LICENSE /go/src/github.com/umputun/remark/
COPY --from=build-frontend /srv/web/public/ web
RUN \
export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
go get -v github.com/rakyll/statik && \
statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
ls -la /go/src/github.com/umputun/remark/backend/web/
# if DRONE presented use DRONE_* git env to make version
RUN \
if [ -z "$DRONE" ] ; then \
echo "runs outside of drone" && version="local"; \
else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
echo "version=$version" && \
GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
RUN \
if [ -z "$DRONE_TAG" ] ; then \
echo "runs outside of drone" && tag=""; \
else tag=_${DRONE_TAG}; fi && \
apk add --no-cache --update zip && \
tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
# upload to github
RUN \
if [ -z "$DRONE_TAG" ] ; then \
echo "skip upload to github" ; \
else \
curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
"https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
"https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
"https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
"https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
"https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
FROM alpine
COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
RUN ls -la /artifacts/*
CMD ["sleep", "100"]
Makefile
+26
View File
@@ -0,0 +1,26 @@
OS=linux
ARCH=amd64
bin:
docker build -f Dockerfile.artifacts -t remark42.bin .
- @docker rm -f remark42.bin 2>/dev/null || exit 0
docker run -d --name=remark42.bin remark42.bin
docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
docker rm -f remark42.bin
docker:
docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
deploy:
docker build -f Dockerfile.artifacts -t remark42.bin .
- @docker rm -f remark42.bin 2>/dev/null || exit 0
- @mkdir -p bin
docker run -d --name=remark42.bin remark42.bin
docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
docker rm -f remark42.bin
.PHONY: bin
README.md
+201 -114
View File
@@ -4,7 +4,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
* Social login via Google, Facebook, Github and Yandex
* Multi-level nested comments with both tree and plain presentations
* Import from disqus
* Import from disqus and wordpress
* Markdown support
* Moderator can remove comments and block users
* Voting, pinning and verification system
@@ -14,57 +14,113 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
* Export data to json with automatic backups
* No external databases, everything embedded in a single data file
* Fully dockerized and can be deployed in a single command
* Self-contained executable can be deployed directly to Linux, Windows and MacOS
* Clean, lightweight and fully customizable UI
* Multi-site mode from a single instance
* Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le)
* [Privacy focused](#privacy)
#
- [Install](#install)
- [Backend](#backend)
- [With Docker](#with-docker)
- [Without docker](#without-docker)
- [Parameters](#parameters)
- [Required parameters](#required-parameters)
- [Register oauth2 providers](#register-oauth2-providers)
- [Google Auth Provider](#google-auth-provider)
- [GitHub Auth Provider](#github-auth-provider)
- [Facebook Auth Provider](#facebook-auth-provider)
- [Yandex Auth Provider](#yandex-auth-provider)
- [Initial import from Disqus](#initial-import-from-disqus)
- [Initial import from WordPress](#initial-import-from-wordpress)
- [Backup and restore](#backup-and-restore)
- [Automatic backups](#automatic-backups)
- [Manual backup](#manual-backup)
- [Restore from backup](#restore-from-backup)
- [Backup format](#backup-format)
- [Admin users](#admin-users)
- [Setup on your website](#setup-on-your-website)
- [Comments](#comments)
- [Last comments](#last-comments)
- [Counter](#counter)
- [Build from the source](#build-from-the-source)
- [Development](#development)
- [Backend development](#backend-development)
- [Frontend development](#frontend-development)
- [Build](#build)
- [Devserver](#devserver)
- [API](#api)
- [Authorization](#authorization)
- [Commenting](#commenting)
- [RSS feeds](#rss-feeds)
- [Admin](#admin)
- [Privacy](#privacy)
- [Technical details](#technical-details)
## Install
### Backend
#### With Docker
_this is the recommended way to run remark42_
* copy provided `docker-compose.yml` and customize for your needs
* prepare user id for container `` export USER=`id -u $USER` ``
* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
* alternatively compile from sources - `docker-compose build && docker-compose up -d`
* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
#### Without docker
* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
* unpack with `gunzip` (linux, mac os) or with `zip` (windows)
* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
#### Parameters
| Command line | Environment | Default | Description |
| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
| url | REMARK_URL | | url to remark42 server, _required_ |
| secret | SECRET | | secret key, _required_ |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| site | SITE | `remark` | site name(s), _multi_ |
| admin | ADMIN | | admin names (list of user ids), _multi_ |
| admin-email | ADMIN_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatar.path | AVATAR_FS_PATH | `./var/avatars` | avatars location |
| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit |
| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL |
| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL |
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret |
| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only |
| low-score | LOW_SCORE | `-5` | low score threshold |
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
| edit-time | EDIT_TIME | `5m` | edit window |
| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images |
| dbg | DEBUG | `false` | debug mode |
| dev-passwd | DEV_PASSWD | | password for `dev` user |
| Command line | Environment | Default | Description |
| ------------------ | ------------------ | --------------------- | ------------------------------------------------ |
| url | REMARK_URL | | url to remark42 server, _required_ |
| secret | SECRET | | secret key, _required_ |
| site | SITE | `remark` | site name(s), _multi_ |
| store.type | STORE_TYPE | `bolt` | type of storage, `bolt` or `mongo` |
| store.bolt.path | STORE_BOLT_PATH | `./var` | path to data directory |
| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s` | boltdb access timeout |
| mongo.url | MONGO_URL | | mongo url for all stores using mongodb |
| mongo.db | MONGO_DB | | mongo database |
| admin.shared.id | ADMIN_SHARED_ID | | admin names (list of user ids), _multi_ |
| admin.shared.email | ADMIN_SHARED_EMAIL | `admin@${REMARK_URL}` | admin email |
| backup | BACKUP_PATH | `./var/backup` | backups location |
| max-back | MAX_BACKUP_FILES | `10` | max backup files to keep |
| cache.max.items | CACHE_MAX_ITEMS | `1000` | max number of cached items, `0` - unlimited |
| cache.max.value | CACHE_MAX_VALUE | `65536` | max size of cached value, `0` - unlimited |
| cache.max.size | CACHE_MAX_SIZE | `50000000` | max size of all cached values, `0` - unlimited |
| avatar.type | AVATAR_TYPE | `fs` | type of avatar storage, `fs`, 'bolt`, or `mongo` |
| avatar.fs.path | AVATAR_FS_PATH | `./var/avatars` | avatars location for `fs` store |
| avatar.bolt.file | AVATAR_BOLT_FILE | `./var/avatars.db` | file name for `bolt` store |
| avatar.rsz-lmt | AVATAR_RSZ_LMT | 0 | max image size for resizing avatars on save |
| auth.ttl.jwt | AUTH_TTL_JWT | 5m | jwt TTL |
| auth.ttl.cookie | AUTH_TTL_COOKIE | 200h | cookie TTL |
| auth.google.cid | AUTH_GOOGLE_CID | | Google OAuth client ID |
| auth.google.csec | AUTH_GOOGLE_CSEC | | Google OAuth client secret |
| auth.facebook.cid | AUTH_FACEBOOK_CID | | Facebook OAuth client ID |
| auth.facebook.csec | AUTH_FACEBOOK_CSEC | | Facebook OAuth client secret |
| auth.github.cid | AUTH_GITHUB_CID | | Github OAuth client ID |
| auth.github.csec | AUTH_GITHUB_CSEC | | Github OAuth client secret |
| auth.yandex.cid | AUTH_YANDEX_CID | | Yandex OAuth client ID |
| auth.yandex.csec | AUTH_YANDEX_CSEC | | Yandex OAuth client secret |
| auth.dev | AUTH_DEV | false | local oauth2 server, development mode only |
| max-comment | MAX_COMMENT_SIZE | 2048 | comment's size limit |
| low-score | LOW_SCORE | `-5` | low score threshold |
| critical-score | CRITICAL_SCORE | `-10` | critical score threshold |
| edit-time | EDIT_TIME | `5m` | edit window |
| img-proxy | IMG_PROXY | `false` | enable http->https proxy for images |
| dbg | DEBUG | `false` | debug mode |
| dev-passwd | DEV_PASSWD | | password for `dev` user |
* command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
* _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -75,8 +131,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
1. `SECRET` - secret key, can be any long and hard-to-guess string.
1. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
2. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
The minimal `docker-compose.yml` has to include all required parameters:
@@ -90,10 +146,10 @@ services:
container_name: "remark42"
environment:
- REMARK_URL=https://demo.remark42.com # url pointing to your remark42 server
- SITE=YOUR_SITE_ID # site ID, same as used for `site_id`, see "Setup on your website"
- SECRET=abcd-123456-xyz-$%^& # secret key
- AUTH_GITHUB_CID=12345667890 # oauth2 client ID
- AUTH_GITHUB_CSEC=abcdefg12345678 # oauth2 client secret
- USER=1001 # UID on the host machine, i.e `id -u`
volumes:
- ./var:/srv/var # persistent volume to store all remark42 data
```
@@ -154,7 +210,13 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co
1. Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
2. Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
3. Run import command - `docker-compose exec remark42 /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
3. Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
#### Initial import from WordPress
1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
2. Move this file to your remark42 host within `./var`
3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
#### Backup and restore
@@ -163,20 +225,19 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./
For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}`
##### Schema migration
One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema
and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance.
`docker-compose exec remark42 /srv/migrate-data.sh {your site id}`
`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
##### Manual backup
In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz`
In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
`docker-compose exec remark42 /srv/create-backup.sh {your site id}`
`docker exec -it remark42 backup -s {your site id}`
##### Restore from backup
Restore will clean all comments first and then will processed with complete import from a given file.
`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
##### Backup format
@@ -195,50 +256,9 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
To get user id just login and click on your username or any other user you want to promote to admins.
It will expand login info and show full user ID.
### Setup on your website
### Frontend
Frontend part is building automatically along with backend if you use `docker-compose`.
For manual building:
* install [Node.js 8](https://nodejs.org/en/) or higher;
* install [NPM 6.1.0](https://www.npmjs.com/package/npm) or higher;
* run `npm install` inside `./web`;
* run `npm run build` there;
* result files will be saved in `./web/public`.
For development mode use `npm start` instead of `npm run build`.
In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
URLs for development:
* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
* `localhost:8080/dev.html` — page with embedded script from local folder;
* `localhost:8080/last-comments.html` — page with embedded script for last comments;
* `localhost:8080/counter.html` — page with embedded script for counter with examples.
#### Testing
Also you can use fully functional local version to develop and test both frontend & backend.
To bring it up run:
```bash
docker-compose -f compose-dev-frontend.yml build
docker-compose -f compose-dev-frontend.yml up
```
It starts Remark42 on `localhost:8080`
and adds local OAuth2 provider “Dev”. To access UI demo page go to `localhost:8080/web`.
That `compose-dev.yml` (you can find it in the root of the project) also defines if default logged user admin or not.
By default, it will be the admin, and to switch it to regular user comment or remove `-ADMIN=dev_user` there. You can also select
any other user name from the login dialog.
#### Usage
##### Comments
#### Comments
It's a main widget which renders list of comments.
@@ -268,7 +288,7 @@ And then add this node in the place where you want to see Remark42 widget:
After that widget will be rendered inside this node.
##### Last comments
#### Last comments
It's a widget which renders list of last comments from your site.
@@ -296,7 +316,7 @@ And then add this node in the place where you want to see last comments widget:
`data-max` sets the max amount of comments (default: `15`).
##### Counter
#### Counter
It's a widget which renders a number of comments for the specified page.
@@ -305,7 +325,7 @@ Add this snippet to the bottom of web page:
```html
<script>
var remark_config = {
site_id: 'YOUR_SITE_ID',
site_id: 'YOUR_SITE_ID',
};
(function() {
@@ -328,6 +348,76 @@ and it will use `data-url` attribute to define the page with comments.
Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
## Build from the source
- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
`remark42` file with everything embedded.
## Development
You can use fully functional local version to develop and test both frontend & backend.
To bring it up run:
```bash
# if you mainly work on backend
docker-compose -f compose-dev-backend.yml build
docker-compose -f compose-dev-backend.yml up
# if you mainly work on frontend
docker-compose -f compose-dev-frontend.yml build
docker-compose -f compose-dev-frontend.yml up
```
It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
To access UI demo page go to `127.0.0.1:8080/web`.
By default, you would be logged in as `dev_user` which defined as admin.
You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
Backend docker compose config by default skips running frontend related tests.
Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
### Backend development
In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
### Frontend development
#### Build
* install [Node.js 8](https://nodejs.org/en/) or higher;
* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
* run `npm install` inside `./web`;
* run `npm run build` there;
* result files will be saved in `./web/public`.
**Note** Running `npm install` will set up precommit hooks into your git repository.
It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
#### Devserver
For local development mode with Hot Reloading use `npm start` instead of `npm run build`.
In this case `webpack` will serve files using `webpack-dev-server` on `localhost:9000`.
By visiting `127.0.0.1:9000/web` you will get a page with main comments widget.
communicating with demo server backend running on `https://demo.remark42.com`.
But you will not be able to login with any oauth providers due to security reasons.
You can attach to locally running backend by providing `REMARK_URL` environment variable.
```sh
npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
```
Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
[Redux](https://github.com/zalmoxisus/redux-devtools-extension).
## API
### Authorization
@@ -394,15 +484,14 @@ type Node struct {
Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in 5min since creation
* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation. Body is `EditRequest` json
```json
Content-Type: application/json
{
"text": "edit comment blah http://radio-t.com 12345",
"summary": "fix blah"
}
```go
type EditRequest struct {
Text string `json:"text"` // updated text
Summary string `json:"summary"` // optional, summary of the edit
Delete bool `json:"delete"` // delete flag
}{}
```
* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
@@ -472,22 +561,20 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
_all admin calls require auth and admin privilege_
## Privacy
* Remark42 is trying to be very sensitive to any private or semi-private information.
* Authentication requesting the lowest (minimal) possible scope from providers. All extra information returned by them dropped immediately and not stored in any form.
* Generally remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
* There is no tracking of any sort.
* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing Cross-Site Request Forgery
* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF.
* There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
* There are no third-party analytic services involved.
* User can request all information remark42 knows about and export to gz file.
* Supported complete cleanup of all information related to user activity on demand.
* Supported complete cleanup of all information related to user's activity.
* Cookie lifespan can be restricted to session-only.
* All potentially sensitive data stored by remark42 hashed and encrypted.
## Technical details
* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `STORE_BOLT_PATH`
@@ -500,8 +587,8 @@ _all admin calls require auth and admin privilege_
* Request timeout set to 60sec
* Development mode (`--dev-password` set) allows to test remark42 without social login and with admin privileges. Adds basic-auth for username: `dev`, password: `${DEV_PASSWD}`. **should not be used in production deployment**
* User can vote for the comment multiple times but only to change the vote. Double-voting not allowed.
* User can edit comments in 5 mins window after creation.
* User can edit comments in 5 mins (configurable) window after creation.
* User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
* All avatars resized and cached locally to prevent rate limiters from google/github/facebook/yandex.
* All avatars resized and cached locally to prevent rate limiters from oauth providers.
* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
* Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.
backend/Gopkg.lock
Generated
+162 -9
View File
@@ -2,185 +2,295 @@
[[projects]]
digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
name = "cloud.google.com/go"
packages = ["compute/metadata"]
pruneopts = "UT"
revision = "767c40d6a2e058483c25fa193e963a22da17236d"
version = "v0.18.0"
[[projects]]
digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
name = "github.com/PuerkitoBio/goquery"
packages = ["."]
pruneopts = "UT"
revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
version = "v1.4.0"
[[projects]]
digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
name = "github.com/andybalholm/cascadia"
packages = ["."]
pruneopts = "UT"
revision = "901648c87902174f774fac311d7f176f8647bdaa"
version = "v1.0.0"
[[projects]]
digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
name = "github.com/coreos/bbolt"
packages = ["."]
pruneopts = "UT"
revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
version = "v1.3.0"
[[projects]]
digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = "UT"
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
name = "github.com/dgrijalva/jwt-go"
packages = ["."]
pruneopts = "UT"
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
version = "v3.2.0"
[[projects]]
digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
name = "github.com/didip/tollbooth"
packages = [
".",
"errors",
"libstring",
"limiter"
"limiter",
]
pruneopts = "UT"
revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
version = "v4.0.0"
[[projects]]
branch = "master"
digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
name = "github.com/didip/tollbooth_chi"
packages = ["."]
pruneopts = "UT"
revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
[[projects]]
digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
name = "github.com/globalsign/mgo"
packages = [
".",
"bson",
"internal/json",
"internal/sasl",
"internal/scram",
]
pruneopts = "UT"
revision = "113d3961e7311526535a1ef7042196563d442761"
version = "r2018.06.15"
[[projects]]
digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
name = "github.com/go-chi/chi"
packages = [
".",
"middleware"
"middleware",
]
pruneopts = "UT"
revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
version = "v3.3.2"
[[projects]]
digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
name = "github.com/go-chi/cors"
packages = ["."]
pruneopts = "UT"
revision = "dba6525398619dead495962a916728e7ee2ca322"
version = "v1.0.0"
[[projects]]
digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
name = "github.com/go-chi/render"
packages = ["."]
pruneopts = "UT"
revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
version = "v1.0.0"
[[projects]]
digest = "1:1212e114344a5cdcc834ea69e19d456eef230f9784659080fee67e02ba2cb574"
name = "github.com/go-pkgz/mongo"
packages = ["."]
pruneopts = "UT"
revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c"
version = "v1.0.0"
[[projects]]
digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
name = "github.com/go-pkgz/repeater"
packages = [
".",
"strategy",
]
pruneopts = "UT"
revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
version = "v1.0.0"
[[projects]]
digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
name = "github.com/golang/protobuf"
packages = ["proto"]
pruneopts = "UT"
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
[[projects]]
digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
name = "github.com/google/uuid"
packages = ["."]
pruneopts = "UT"
revision = "064e2069ce9c359c118179501254f67d7d37ba24"
version = "0.2"
[[projects]]
digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
name = "github.com/gorilla/feeds"
packages = ["."]
pruneopts = "UT"
revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
version = "v1.1.0"
[[projects]]
branch = "master"
digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
name = "github.com/hashicorp/errwrap"
packages = ["."]
pruneopts = "UT"
revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
[[projects]]
branch = "master"
digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
name = "github.com/hashicorp/go-multierror"
packages = ["."]
pruneopts = "UT"
revision = "b7773ae218740a7be65057fc60b366a49b538a44"
[[projects]]
branch = "master"
digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
name = "github.com/hashicorp/golang-lru"
packages = [
".",
"simplelru"
"simplelru",
]
pruneopts = "UT"
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
[[projects]]
branch = "master"
digest = "1:ce3f7860fd68bd2dd4c3735e2aed8c9de7c7d05bd6ad7d97a6bedcf4fe7b84fb"
name = "github.com/hashicorp/logutils"
packages = ["."]
pruneopts = "UT"
revision = "0dc08b1671f34c4250ce212759ebd880f743d883"
[[projects]]
branch = "master"
digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
name = "github.com/jessevdk/go-flags"
packages = ["."]
pruneopts = "UT"
revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
[[projects]]
branch = "master"
digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
name = "github.com/microcosm-cc/bluemonday"
packages = ["."]
pruneopts = "UT"
revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
[[projects]]
branch = "master"
digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
name = "github.com/nullrocks/identicon"
packages = ["."]
pruneopts = "UT"
revision = "7875f45b0022edded6377e40639d8aa620193a62"
[[projects]]
digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
name = "github.com/patrickmn/go-cache"
packages = ["."]
pruneopts = "UT"
revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
version = "v2.1.0"
[[projects]]
digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = "UT"
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = "UT"
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
name = "github.com/rakyll/statik"
packages = ["fs"]
pruneopts = "UT"
revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
version = "v0.1.3"
[[projects]]
branch = "master"
digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
name = "github.com/shurcooL/sanitized_anchor_name"
packages = ["."]
pruneopts = "UT"
revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
[[projects]]
digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
name = "github.com/stretchr/testify"
packages = [
"assert",
"require"
"require",
]
pruneopts = "UT"
revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
version = "v1.2.1"
[[projects]]
branch = "master"
digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
name = "golang.org/x/image"
packages = [
"draw",
"math/f64"
"math/f64",
]
pruneopts = "UT"
revision = "af66defab954cb421ca110193eed9477c8541e2a"
[[projects]]
branch = "master"
digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
name = "golang.org/x/net"
packages = [
"context",
"context/ctxhttp",
"html",
"html/atom"
"html/atom",
]
pruneopts = "UT"
revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
[[projects]]
branch = "master"
digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
name = "golang.org/x/oauth2"
packages = [
".",
@@ -190,23 +300,29 @@
"internal",
"jws",
"jwt",
"yandex"
"yandex",
]
pruneopts = "UT"
revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
[[projects]]
branch = "master"
digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
name = "golang.org/x/sys"
packages = ["unix"]
pruneopts = "UT"
revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
[[projects]]
branch = "master"
digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
name = "golang.org/x/time"
packages = ["rate"]
pruneopts = "UT"
revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
[[projects]]
digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
name = "google.golang.org/appengine"
packages = [
".",
@@ -218,20 +334,57 @@
"internal/modules",
"internal/remote_api",
"internal/urlfetch",
"urlfetch"
"urlfetch",
]
pruneopts = "UT"
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
name = "gopkg.in/russross/blackfriday.v2"
packages = ["."]
pruneopts = "UT"
revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
version = "v2.0.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "af8b7f1817ce6e82746722a745184bd50c341733bcc024311aa956ea32475796"
input-imports = [
"github.com/PuerkitoBio/goquery",
"github.com/coreos/bbolt",
"github.com/dgrijalva/jwt-go",
"github.com/didip/tollbooth",
"github.com/didip/tollbooth_chi",
"github.com/globalsign/mgo",
"github.com/globalsign/mgo/bson",
"github.com/go-chi/chi",
"github.com/go-chi/chi/middleware",
"github.com/go-chi/cors",
"github.com/go-chi/render",
"github.com/go-pkgz/mongo",
"github.com/go-pkgz/repeater",
"github.com/google/uuid",
"github.com/gorilla/feeds",
"github.com/hashicorp/go-multierror",
"github.com/hashicorp/golang-lru",
"github.com/hashicorp/logutils",
"github.com/jessevdk/go-flags",
"github.com/microcosm-cc/bluemonday",
"github.com/nullrocks/identicon",
"github.com/patrickmn/go-cache",
"github.com/pkg/errors",
"github.com/rakyll/statik/fs",
"github.com/stretchr/testify/assert",
"github.com/stretchr/testify/require",
"golang.org/x/image/draw",
"golang.org/x/oauth2",
"golang.org/x/oauth2/facebook",
"golang.org/x/oauth2/github",
"golang.org/x/oauth2/google",
"golang.org/x/oauth2/yandex",
"gopkg.in/russross/blackfriday.v2",
]
solver-name = "gps-cdcl"
solver-version = 1
backend/Gopkg.toml
+14 -6
View File
@@ -24,13 +24,21 @@ required = ["github.com/patrickmn/go-cache"]
name = "gopkg.in/russross/blackfriday.v2"
version = "2.0.0"
[[constraint]]
name = "github.com/patrickmn/go-cache"
version = "2.1.0"
[[constraint]]
branch = "master"
name = "github.com/jessevdk/go-flags"
[[constraint]]
name = "github.com/globalsign/mgo"
version = "r2018.06.15"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
name = "github.com/patrickmn/go-cache"
version = "2.1.0"
[[constraint]]
branch = "master"
name = "github.com/jessevdk/go-flags"
backend/app/cmd/avatar.go
+101
View File
@@ -0,0 +1,101 @@
package cmd
import (
"log"
"path"
"time"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store/avatar"
)
// AvatarCommand set of flags and command for avatar migration
// it converts all avatars from src.type to dst.type.
// Note: it is possible to run migration for the same types (src = dst) in order to resize all avatars.
type AvatarCommand struct {
AvatarSrc AvatarGroup `group:"src" namespace:"src"`
AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
migrator AvatarMigrator
CommonOpts
}
// AvatarMigrator defines interface for migration
type AvatarMigrator interface {
Migrate(avatar.Store, avatar.Store) (int, error)
}
type avatarMigrator struct{}
func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
return avatar.Migrate(dst, src)
}
// Execute runs with AvatarCommand parameters, entry point for "avatar" command
func (ac *AvatarCommand) Execute(args []string) error {
log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
src, err := ac.makeAvatarStore(ac.AvatarSrc)
if err != nil {
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
}
dst, err := ac.makeAvatarStore(ac.AvatarDst)
if err != nil {
return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
}
if ac.migrator == nil {
ac.migrator = avatarMigrator{}
}
count, err := ac.migrator.Migrate(dst, src)
if err != nil {
return err
}
if err = dst.Close(); err != nil {
log.Printf("[WARN] failed to close dst store %s", ac.AvatarDst.Type)
}
if err = src.Close(); err != nil {
log.Printf("[WARN] failed to close src store %s", ac.AvatarSrc.Type)
}
log.Printf("[INFO] completed, migrated avatars = %d", count)
return nil
}
func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
log.Printf("[DEBUG] make avatar store, type=%s", gr.Type)
switch gr.Type {
case "fs":
if err := makeDirs(gr.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(gr.FS.Path, gr.RszLmt), nil
case "mongo":
mgServer, err := ac.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
return avatar.NewGridFS(conn, gr.RszLmt), nil
case "bolt":
if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
return nil, err
}
return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{}, gr.RszLmt)
}
return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
}
func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
if ac.Mongo.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
}
backend/app/cmd/avatar_test.go
+65
View File
@@ -0,0 +1,65 @@
package cmd
import (
"errors"
"os"
"testing"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/avatar"
)
func TestAvatar_Execute(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
defer os.RemoveAll("/tmp/ava-test")
// from fs to mongo
cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
// from fs to bolt
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
"--dst.bolt.file=/tmp/ava-test.db"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
// failed
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.Error(t, err, "failed blah")
}
type avatarMigratorMock struct {
called int
retError error
retCount int
}
func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
a.called++
return a.retCount, a.retError
}
backend/app/cmd/backup.go
+79
View File
@@ -0,0 +1,79 @@
package cmd
import (
"context"
"fmt"
"io"
"log"
"net/http"
"os"
"time"
"github.com/pkg/errors"
)
// BackupCommand set of flags and command for export
// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
type BackupCommand struct {
ExportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
ExportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
Site string `long:"site" env:"SITE" default:"remark" description:"site name"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
CommonOpts
}
// Execute runs export with ExportCommand parameters, entry point for "export" command
func (ec *BackupCommand) Execute(args []string) error {
log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
resetEnv("SECRET")
fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile}
fname, err := fp.parse(time.Now())
if err != nil {
return err
}
log.Printf("[DEBUG] export file %s", fname)
// prepare http client and request
client := http.Client{}
ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
defer cancel()
exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s&secret=%s", ec.RemarkURL, ec.Site, ec.SharedSecret)
req, err := http.NewRequest(http.MethodGet, exportURL, nil)
if err != nil {
return errors.Wrapf(err, "can't make export request for %s", exportURL)
}
// get with timeout
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrapf(err, "request failed for %s", exportURL)
}
defer func() {
if err = resp.Body.Close(); err != nil {
log.Printf("[WARN] failed to close response, %s", err)
}
}()
if resp.StatusCode >= 300 {
return responseError(resp)
}
fh, err := os.Create(fname)
if err != nil {
return errors.Wrapf(err, "can't create backup file %s", fname)
}
defer func() {
if err = fh.Close(); err != nil {
log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
}
}()
if _, err = io.Copy(fh, resp.Body); err != nil {
return errors.Wrapf(err, "failed to write backup file %s", fname)
}
log.Printf("[INFO] export completed, file %s", fname)
return nil
}
backend/app/cmd/backup_test.go
+73
View File
@@ -0,0 +1,73 @@
package cmd
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBackup_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
fmt.Fprint(w, "blah\nblah2\n12345678\n")
}))
defer ts.Close()
cmd := BackupCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
defer os.Remove("/tmp/remark-test.export")
data, err := ioutil.ReadFile("/tmp/remark-test.export")
require.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
}
func TestBackup_ExecuteFailedStatus(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
w.WriteHeader(400)
fmt.Fprint(w, "some error")
}))
defer ts.Close()
cmd := BackupCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.EqualError(t, err, `error response "400 Bad Request", some error`)
}
func TestBackup_ExecuteFailedWrite(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
assert.Equal(t, "GET", r.Method)
fmt.Fprint(w, "blah\nblah2\n12345678\n")
}))
defer ts.Close()
cmd := BackupCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file=/tmp/no-such-dir/{{.SITE}}-test.export"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
}
backend/app/cmd/cmd.go
+129
View File
@@ -0,0 +1,129 @@
// Package cmd has all top-level commands dispatched by main's flag.Parse
// The entry point of each command is Execute function
package cmd
import (
"bytes"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"text/template"
"time"
"github.com/pkg/errors"
)
// CommonOptionsCommander extends flags.Commander with SetCommon
// All commands should implement this interfaces
type CommonOptionsCommander interface {
SetCommon(commonOpts CommonOpts)
Execute(args []string) error
}
// CommonOpts sets externally from main, shared across all commands
type CommonOpts struct {
RemarkURL string
SharedSecret string
Revision string
}
// SetCommon satisfies CommonOptionsCommander interface and sets common option fields
// The method called by main for each command
func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
c.RemarkURL = commonOpts.RemarkURL
c.SharedSecret = commonOpts.SharedSecret
c.Revision = commonOpts.Revision
}
// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
type fileParser struct {
site string
file string
path string
}
// parse apply template and also concat path and file. In case if file contains path separator path will be ignored
func (p *fileParser) parse(now time.Time) (string, error) {
// file/location parameters my have template masks
fileTemplate := struct {
YYYYMMDD string
YYYY string
YYYYMM string
MM string
DD string
TS string
UNIX int64
SITE string
}{
YYYYMMDD: now.Format("20060102"),
YYYY: now.Format("2006"),
YYYYMM: now.Format("200601"),
MM: now.Format("01"),
DD: now.Format("02"),
UNIX: now.Unix(),
SITE: p.site,
TS: now.Format("20060102T150405"),
}
bb := bytes.Buffer{}
fname := p.file
if !strings.Contains(p.file, string(filepath.Separator)) {
fname = filepath.Join(p.path, p.file)
}
if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
return "", errors.Wrapf(err, "failed to parse %q", fname)
}
return bb.String(), nil
}
// resetEnv clears sensitive env vars
func resetEnv(envs ...string) {
for _, env := range envs {
if err := os.Unsetenv(env); err != nil {
log.Printf("[WARN] can't unset env %s, %s", env, err)
}
}
}
// responseError returns error with status and response body
func responseError(resp *http.Response) error {
body, e := ioutil.ReadAll(resp.Body)
if e != nil {
body = []byte("")
}
return errors.Errorf("error response %q, %s", resp.Status, body)
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
for _, dir := range dirs {
ex, err := exists(dir)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
backend/app/cmd/cmd_test.go
+39
View File
@@ -0,0 +1,39 @@
package cmd
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestExport_ParseFileName(t *testing.T) {
tbl := []struct {
p fileParser
res string
err bool
}{
{fileParser{}, "", false},
{fileParser{path: "/tmp/blah", file: "fname.gz"}, "/tmp/blah/fname.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYYMMDD}}.gz"},
"/tmp/blah/fname-remark-20180821.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
"/tmp/blah/fname-remark-2018-08.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
"/tmp/fname-remark-2018-08.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.TS}}.gz"},
"/tmp/fname-remark-20180821T212615.gz", false},
{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.XXX}}-{{.YYYY}}-{{.MM}}.gz"},
"", true},
}
now := time.Date(2018, 8, 21, 21, 26, 15, 0, time.UTC)
for i, tt := range tbl {
r, err := tt.p.parse(now)
if tt.err {
assert.NotNil(t, err)
continue
}
assert.Equal(t, tt.res, r, "check #%d", i)
}
}
backend/app/cmd/import.go
+83
View File
@@ -0,0 +1,83 @@
package cmd
import (
"compress/gzip"
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"time"
"github.com/pkg/errors"
)
// ImportCommand set of flags and command for import
type ImportCommand struct {
InputFile string `short:"f" long:"file" description:"input file name" required:"true"`
Provider string `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
Site string `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
CommonOpts
}
// Execute runs import with ImportCommand parameters, entry point for "import" command
func (ic *ImportCommand) Execute(args []string) error {
log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
resetEnv("SECRET")
reader, err := ic.reader(ic.InputFile)
if err != nil {
return errors.Wrapf(err, "can't open import file %s", ic.InputFile)
}
client := http.Client{}
ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
defer cancel()
importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s&secret=%s",
ic.RemarkURL, ic.Site, ic.Provider, ic.SharedSecret)
req, err := http.NewRequest(http.MethodPost, importURL, reader)
if err != nil {
return errors.Wrapf(err, "can't make import request for %s", importURL)
}
resp, err := client.Do(req.WithContext(ctx)) // closes request's reader
if err != nil {
return errors.Wrapf(err, "request failed for %s", importURL)
}
defer func() {
if err = resp.Body.Close(); err != nil {
log.Printf("[WARN] failed to close response, %s", err)
}
}()
if resp.StatusCode >= 300 {
return responseError(resp)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "can't get response from importer")
}
log.Printf("[INFO] import completed, status=%d, %s", resp.StatusCode, string(body))
return nil
}
// reader returns reader for file. For .gz file wraps with gunzip
func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
inpFile, err := os.Open(inp)
if err != nil {
return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
}
reader = inpFile
if strings.HasSuffix(ic.InputFile, ".gz") {
if reader, err = gzip.NewReader(inpFile); err != nil {
return nil, errors.Wrap(err, "can't make gz reader")
}
}
return reader, nil
}
backend/app/cmd/import_test.go
+119
View File
@@ -0,0 +1,119 @@
package cmd
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestImport_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
cmd = ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
}
func TestImport_ExecuteFailed(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
fmt.Fprintln(w, "some response")
}))
defer ts.Close()
cmd := ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt"})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err, "fail on no such file")
assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
cmd = ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err, "fail on connection refused")
assert.True(t, strings.Contains(err.Error(), "connection refused"))
ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%+v", r)
w.WriteHeader(400)
fmt.Fprintln(w, "some response with 400")
}))
defer ts2.Close()
cmd = ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt"})
require.Nil(t, err)
err = cmd.Execute(nil)
t.Log(err)
assert.NotNil(t, err)
}
func TestImport_ExecuteTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
time.Sleep(500 * time.Millisecond)
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := ImportCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
}
backend/app/cmd/restore.go
+37
View File
@@ -0,0 +1,37 @@
package cmd
import (
"log"
"time"
)
// RestoreCommand set of flags and command for restore from backup
type RestoreCommand struct {
ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
Site string `long:"site" env:"SITE" default:"remark" description:"site name"`
Timeout time.Duration `long:"timeout" default:"15m" description:"import timeout"`
CommonOpts
}
// Execute runs import with RestoreCommand parameters, entry point for "restore" command
// uses ImportCommand with constructed full file name
func (rc *RestoreCommand) Execute(args []string) error {
log.Printf("[INFO] restore %s, site %s", rc.ImportFile, rc.Site)
resetEnv("SECRET")
fp := fileParser{site: rc.Site, path: rc.ImportPath, file: rc.ImportFile}
fname, err := fp.parse(time.Now())
if err != nil {
return err
}
importer := ImportCommand{
InputFile: fname,
Site: rc.Site,
Provider: "native",
Timeout: rc.Timeout,
CommonOpts: rc.CommonOpts,
}
return importer.Execute(args)
}
backend/app/cmd/restore_test.go
+38
View File
@@ -0,0 +1,38 @@
package cmd
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
flags "github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRestore_Execute(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
assert.Equal(t, "POST", r.Method)
assert.Equal(t, "native", r.URL.Query().Get("provider"))
body, err := ioutil.ReadAll(r.Body)
assert.Nil(t, err)
assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
fmt.Fprintln(w, "some response")
fmt.Fprintln(w, string(body))
}))
defer ts.Close()
cmd := RestoreCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt"})
require.Nil(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
}
backend/app/cmd/server.go
+443
View File
@@ -0,0 +1,443 @@
package cmd
import (
"context"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"path"
"strings"
"syscall"
"time"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
// ServerCommand with command line flags and env
type ServerCommand struct {
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
Auth struct {
TTL struct {
JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
CommonOpts
}
// AuthGroup defines options group for auth params
type AuthGroup struct {
CID string `long:"cid" env:"CID" description:"OAuth client ID"`
CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
}
// StoreGroup defines options group for store params
type StoreGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
}
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
Bolt struct {
File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
// MongoGroup holds all mongo params, used by store, avatar and cache
type MongoGroup struct {
URL string `long:"url" env:"URL" description:"mongo url"`
DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
}
// AdminGroup defines options group for admin params
type AdminGroup struct {
Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
Shared struct {
Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
Email string `long:"email" env:"EMAIL" default:"" description:"admin email"`
} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
}
// serverApp holds all active objects
type serverApp struct {
*ServerCommand
restSrv *api.Rest
migratorSrv *api.Migrator
exporter migrator.Exporter
devAuth *auth.DevAuthServer
dataService *service.DataStore
avatarStore avatar.Store
terminated chan struct{}
}
// Execute is the entry point for "server" command, called by flag parser
func (s *ServerCommand) Execute(args []string) error {
log.Printf("[INFO] start server on port %d", s.Port)
resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
ctx, cancel := context.WithCancel(context.Background())
go func() { // catch signal and invoke graceful termination
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
log.Print("[WARN] interrupt signal")
cancel()
}()
app, err := s.newServerApp()
if err != nil {
log.Fatalf("[ERROR] failed to setup application, %+v", err)
}
if err = app.run(ctx); err != nil {
log.Printf("[WARN] remark terminated with error %+v", err)
return err
}
log.Printf("[INFO] remark terminated")
return nil
}
// newServerApp prepares application and return it with all active parts
// doesn't start anything
func (s *ServerCommand) newServerApp() (*serverApp, error) {
if err := makeDirs(s.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL)
}
log.Printf("[INFO] root url=%s", s.RemarkURL)
storeEngine, err := s.makeDataStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make data store engine")
}
adminStore, err := s.makeAdminStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make admin store")
}
dataService := &service.DataStore{
Interface: storeEngine,
EditDuration: s.EditDuration,
AdminStore: adminStore,
MaxCommentSize: s.MaxCommentSize,
}
loadingCache, err := s.makeCache()
if err != nil {
return nil, errors.Wrap(err, "failed to make cache")
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(adminStore, strings.HasPrefix(s.RemarkURL, "https://"), s.Auth.TTL.JWT, s.Auth.TTL.Cookie)
avatarStore, err := s.makeAvatarStore()
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(s.RemarkURL, "/"),
}
exporter := &migrator.Remark{DataStore: dataService}
migr := &api.Migrator{
Cache: loadingCache,
NativeImporter: &migrator.Remark{DataStore: dataService},
DisqusImporter: &migrator.Disqus{DataStore: dataService},
WordPressImporter: &migrator.WordPress{DataStore: dataService},
NativeExported: &migrator.Remark{DataStore: dataService},
KeyStore: adminStore,
}
authProviders := s.makeAuthProviders(jwtService, avatarProxy, dataService)
imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
commentFormatter := store.NewCommentFormatter(imgProxy)
srv := &api.Rest{
Version: s.Revision,
DataService: dataService,
WebRoot: s.WebRoot,
RemarkURL: s.RemarkURL,
ImageProxy: imgProxy,
CommentFormatter: commentFormatter,
AvatarProxy: avatarProxy,
Migrator: migr,
ReadOnlyAge: s.ReadOnlyAge,
SharedSecret: s.SharedSecret,
Authenticator: auth.Authenticator{
JWTService: jwtService,
KeyStore: adminStore,
Providers: authProviders,
DevPasswd: s.DevPasswd,
PermissionChecker: dataService,
},
Cache: loadingCache,
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
var devAuth *auth.DevAuthServer
if s.Auth.Dev {
devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]}
}
return &serverApp{
ServerCommand: s,
restSrv: srv,
migratorSrv: migr,
exporter: exporter,
devAuth: devAuth,
dataService: dataService,
avatarStore: avatarStore,
terminated: make(chan struct{}),
}, nil
}
// Run all application objects
func (a *serverApp) run(ctx context.Context) error {
if a.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
go func() {
// shutdown on context cancellation
<-ctx.Done()
a.restSrv.Shutdown()
if a.devAuth != nil {
a.devAuth.Shutdown()
}
if e := a.dataService.Close(); e != nil {
log.Printf("[WARN] failed to close data store, %s", e)
}
if e := a.avatarStore.Close(); e != nil {
log.Printf("[WARN] failed to close avatar store, %s", e)
}
}()
a.activateBackup(ctx) // runs in goroutine for each site
if a.Auth.Dev {
go a.devAuth.Run() // dev oauth2 server on :8084
}
a.restSrv.Run(a.Port)
close(a.terminated)
return nil
}
// Wait for application completion (termination)
func (a *serverApp) Wait() {
<-a.terminated
}
// activateBackup runs background backups for each site
func (a *serverApp) activateBackup(ctx context.Context) {
for _, siteID := range a.Sites {
backup := migrator.AutoBackup{
Exporter: a.exporter,
BackupLocation: a.BackupLocation,
SiteID: siteID,
KeepMax: a.MaxBackupFiles,
Duration: 24 * time.Hour,
}
go backup.Do(ctx)
}
}
// makeDataStore creates store for all sites
func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
log.Printf("[INFO] make data store, type=%s", s.Store.Type)
switch s.Store.Type {
case "bolt":
if err = makeDirs(s.Store.Bolt.Path); err != nil {
return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
for _, site := range s.Sites {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
case "mongo":
mgServer, e := s.makeMongo()
if e != nil {
return result, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
}
return result, errors.Wrap(err, "can't initialize data store")
}
func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
log.Printf("[INFO] make avatar store, type=%s", s.Avatar.Type)
switch s.Avatar.Type {
case "fs":
if err := makeDirs(s.Avatar.FS.Path); err != nil {
return nil, err
}
return avatar.NewLocalFS(s.Avatar.FS.Path, s.Avatar.RszLmt), nil
case "mongo":
mgServer, err := s.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
return avatar.NewGridFS(conn, s.Avatar.RszLmt), nil
case "bolt":
if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
return nil, err
}
return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{}, s.Avatar.RszLmt)
}
return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
}
func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
switch s.Admin.Type {
case "shared":
if s.Admin.Shared.Email == "" { // no admin email, use admin@domain
if u, err := url.Parse(s.RemarkURL); err == nil {
s.Admin.Shared.Email = "admin@" + u.Host
}
}
return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
case "mongo":
mgServer, e := s.makeMongo()
if e != nil {
return nil, errors.Wrap(e, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
return admin.NewMongoStore(conn), nil
default:
return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
}
}
func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
switch s.Cache.Type {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
case "mongo":
mgServer, err := s.makeMongo()
if err != nil {
return nil, errors.Wrap(err, "failed to create mongo server")
}
conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
}
return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
}
func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
if s.Mongo.URL == "" {
return nil, errors.New("no mongo URL provided")
}
return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
}
func (s *ServerCommand) makeAuthProviders(jwt *auth.JWT, ap *proxy.Avatar, ds *service.DataStore) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwt,
AvatarProxy: ap,
RemarkURL: s.RemarkURL,
Cid: cid,
Csecret: secret,
PermissionChecker: ds,
}
}
providers := []auth.Provider{}
if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(s.Auth.Google.CID, s.Auth.Google.CSEC)))
}
if s.Auth.Github.CID != "" && s.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(s.Auth.Github.CID, s.Auth.Github.CSEC)))
}
if s.Auth.Facebook.CID != "" && s.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(s.Auth.Facebook.CID, s.Auth.Facebook.CSEC)))
}
if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)))
}
if s.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
backend/app/cmd/server_test.go
+246
View File
@@ -0,0 +1,246 @@
package cmd
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"syscall"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestServerApp(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18080
return o
})
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18080/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
// add comment
resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
require.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
body, _ = ioutil.ReadAll(resp.Body)
t.Log(string(body))
assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
app.Wait()
}
func TestServerApp_DevMode(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18085
o.DevPasswd = "password"
o.Auth.Dev = true
return o
})
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider")
assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider")
// send ping
resp, err := http.Get("http://localhost:18085/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestServerApp_WithMongo(t *testing.T) {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://localhost:27017/test"
}
if mongoURL == "skip" {
t.Skip("skip mongo app test")
}
opts := ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--dev-passwd=password", "--cache.type=mongo", "--store.type=mongo",
"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
require.Nil(t, err)
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
// create app
app, err := opts.newServerApp()
require.Nil(t, err)
defer func() {
s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
assert.NoError(t, err)
conn := mongo.NewConnection(s, "test_remark", "")
_ = conn.WithDB(func(dbase *mgo.Database) error {
assert.NoError(t, dbase.DropDatabase())
return nil
})
}()
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(5 * time.Second)
log.Print("[TEST] terminate app")
cancel()
}()
go func() { _ = app.run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:12345/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestServerApp_Failed(t *testing.T) {
opts := ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = opts.newServerApp()
assert.EqualError(t, err, "failed to make data store engine: can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = opts.newServerApp()
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = opts.newServerApp()
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
assert.NotNil(t, err, "blah is invalid type")
opts.Store.Type = "blah"
_, err = opts.newServerApp()
assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah")
t.Log(err)
}
func TestServerApp_Shutdown(t *testing.T) {
app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
o.Port = 18090
return o
})
st := time.Now()
err := app.run(ctx)
assert.Nil(t, err)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
app.Wait()
}
func TestServerApp_MainSignal(t *testing.T) {
go func() {
time.Sleep(100 * time.Millisecond)
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
require.Nil(t, err)
}()
st := time.Now()
s := ServerCommand{}
s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
p := flags.NewParser(&s, flags.Default)
args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100"}
defer os.Remove("/tmp/ava-test.db")
_, err := p.ParseArgs(args)
require.Nil(t, err)
err = s.Execute(args)
assert.NoError(t, err, "execute failed")
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
}
func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
cmd := ServerCommand{}
cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
// prepare options
p := flags.NewParser(&cmd, flags.Default)
_, err := p.ParseArgs([]string{"--dev-passwd=password"})
require.Nil(t, err)
cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
cmd.Store.Bolt.Timeout = 10 * time.Second
cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
cmd.BackupLocation = "/tmp"
cmd = fn(cmd)
os.Remove(cmd.Store.Bolt.Path + "/remark.db")
// create app
app, err := cmd.newServerApp()
require.Nil(t, err)
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(duration)
log.Print("[TEST] terminate app")
cancel()
}()
return app, ctx
}
backend/app/cmd/testdata/import.txt
Vendored
+3
View File
@@ -0,0 +1,3 @@
blah
blah2
12345678
backend/app/cmd/testdata/import.txt.gz
Vendored
BIN
View File
Binary file not shown.
backend/app/main.go
+28 -355
View File
@@ -1,385 +1,58 @@
package main
import (
"context"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/coreos/bbolt"
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
"github.com/umputun/remark/backend/app/cmd"
)
// Opts with command line flags and env
// nolint:maligned
// Opts with all cli commands and flags
type Opts struct {
SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
ServerCmd cmd.ServerCommand `command:"server"`
ImportCmd cmd.ImportCommand `command:"import"`
BackupCmd cmd.BackupCommand `command:"backup"`
RestoreCmd cmd.RestoreCommand `command:"restore"`
AvatarCmd cmd.AvatarCommand `command:"avatar"`
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
Sites []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
Admins []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
AdminEmail string `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
DevPasswd string `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
BackupLocation string `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
MaxBackupFiles int `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
ImageProxy bool `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
MaxCommentSize int `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
LowScore int `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
CriticalScore int `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
ReadOnlyAge int `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
EditDuration time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
Port int `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
WebRoot string `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
Auth struct {
TTL struct {
JWT time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
Google AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
Github AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
Yandex AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
Dev bool `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
}
// AuthGroup defines options group for auth params
type AuthGroup struct {
CID string `long:"cid" env:"CID" description:"OAuth client ID"`
CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
}
// StoreGroup defines options group for store params
type StoreGroup struct {
Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
}
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
type CacheGroup struct {
Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
Size int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
} `group:"max" namespace:"max" env-namespace:"MAX"`
Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
}
var revision = "unknown"
// Application holds all active objects
type Application struct {
Opts
restSrv *api.Rest
migratorSrv *api.Migrator
exporter migrator.Exporter
devAuth *auth.DevAuthServer
terminated chan struct{}
}
func main() {
fmt.Printf("remark %s\n", revision)
fmt.Printf("remark42 %s\n", revision)
var opts Opts
p := flags.NewParser(&opts, flags.Default)
if _, e := p.ParseArgs(os.Args[1:]); e != nil {
os.Exit(1)
}
setupLog(opts.Dbg)
log.Print("[INFO] started remark")
resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
ctx, cancel := context.WithCancel(context.Background())
go func() { // catch signal and invoke graceful termination
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
<-stop
log.Print("[WARN] interrupt signal")
cancel()
}()
app, err := New(opts)
if err != nil {
log.Fatalf("[ERROR] failed to setup application, %+v", err)
}
err = app.Run(ctx)
log.Printf("[INFO] remark terminated %s", err)
}
// New prepares application and return it with all active parts
// doesn't start anything
func New(opts Opts) (*Application, error) {
if err := makeDirs(opts.BackupLocation); err != nil {
return nil, err
}
if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
}
boltStore, err := makeDataStore(opts.Store, opts.Sites)
if err != nil {
return nil, err
}
dataService := &service.DataStore{
Interface: boltStore,
EditDuration: opts.EditDuration,
Secret: opts.SecretKey,
MaxCommentSize: opts.MaxCommentSize,
Admins: opts.Admins,
}
loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
cache.MaxKeys(opts.Cache.Max.Items))
if err != nil {
return nil, err
}
// token TTL is 5 minutes, inactivity interval 7+ days by default
jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"),
opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie)
avatarStore, err := makeAvatarStore(opts.Avatar)
if err != nil {
return nil, errors.Wrap(err, "failed to make avatar store")
}
avatarProxy := &proxy.Avatar{
Store: avatarStore,
RoutePath: "/api/v1/avatar",
RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
}
exporter := &migrator.Remark{DataStore: dataService}
migr := &api.Migrator{
Version: revision,
Cache: loadingCache,
NativeImporter: &migrator.Remark{DataStore: dataService},
DisqusImporter: &migrator.Disqus{DataStore: dataService},
NativeExported: &migrator.Remark{DataStore: dataService},
SecretKey: opts.SecretKey,
}
authProviders := makeAuthProviders(jwtService, avatarProxy, dataService, opts)
srv := &api.Rest{
Version: revision,
DataService: dataService,
Exporter: exporter,
WebRoot: opts.WebRoot,
RemarkURL: opts.RemarkURL,
ImageProxy: &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
AvatarProxy: avatarProxy,
ReadOnlyAge: opts.ReadOnlyAge,
Authenticator: auth.Authenticator{
JWTService: jwtService,
AdminEmail: opts.AdminEmail,
Providers: authProviders,
DevPasswd: opts.DevPasswd,
PermissionChecker: dataService,
},
Cache: loadingCache,
}
// no admin email, use admin@domain
if srv.Authenticator.AdminEmail == "" {
if u, err := url.Parse(opts.RemarkURL); err == nil {
srv.Authenticator.AdminEmail = "admin@" + u.Host
}
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
var devAuth *auth.DevAuthServer
if opts.Auth.Dev {
devAuth = &auth.DevAuthServer{Provider: authProviders[len(authProviders)-1]}
}
tch := make(chan struct{})
return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, Opts: opts, terminated: tch}, nil
}
// Run all application objects
func (a *Application) Run(ctx context.Context) error {
if a.DevPasswd != "" {
log.Printf("[WARN] running in dev mode")
}
go func() {
// shutdown on context cancellation
<-ctx.Done()
a.restSrv.Shutdown()
a.migratorSrv.Shutdown()
if a.devAuth != nil {
a.devAuth.Shutdown()
}
}()
a.activateBackup(ctx) // runs in goroutine for each site
go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only
if a.Auth.Dev {
go a.devAuth.Run() // dev oauth2 server on :8084
}
a.restSrv.Run(a.Port)
close(a.terminated)
return nil
}
// Wait for application completion (termination)
func (a *Application) Wait() {
<-a.terminated
}
// activateBackup runs background backups for each site
func (a *Application) activateBackup(ctx context.Context) {
for _, siteID := range a.Sites {
backup := migrator.AutoBackup{
Exporter: a.exporter,
BackupLocation: a.BackupLocation,
SiteID: siteID,
KeepMax: a.MaxBackupFiles,
Duration: 24 * time.Hour,
}
go backup.Do(ctx)
}
}
// makeDataStore creates store for all sites
func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
switch group.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
return nil, err
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
}
return result, errors.Wrap(err, "can't initialize data store")
}
func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
switch group.Type {
case "fs":
if err = makeDirs(group.FS.Path); err != nil {
return nil, err
}
return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
}
// mkdir -p for all dirs
func makeDirs(dirs ...string) error {
// exists returns whether the given file or directory exists or not
exists := func(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
for _, dir := range dirs {
ex, err := exists(dir)
p.CommandHandler = func(command flags.Commander, args []string) error {
setupLog(opts.Dbg)
// commands implements CommonOptionsCommander to allow passing set of extra options defined for all commands
c := command.(cmd.CommonOptionsCommander)
c.SetCommon(cmd.CommonOpts{
RemarkURL: opts.RemarkURL,
SharedSecret: opts.SharedSecret,
Revision: revision,
})
err := c.Execute(args)
if err != nil {
return errors.Wrapf(err, "can't check directory status for %s", dir)
}
if !ex {
if e := os.MkdirAll(dir, 0700); e != nil {
return errors.Wrapf(err, "can't make directory %s", dir)
}
}
}
return nil
}
func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
return auth.Params{
JwtService: jwtService,
AvatarProxy: avatarProxy,
RemarkURL: opts.RemarkURL,
Cid: cid,
Csecret: secret,
SecretKey: opts.SecretKey,
PermissionChecker: ds,
log.Printf("[ERROR] failed with %+v", err)
}
return err
}
providers := []auth.Provider{}
if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
}
if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
}
if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
}
if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
}
if opts.Auth.Dev {
providers = append(providers, auth.NewDev(makeParams("", "")))
}
if len(providers) == 0 {
log.Printf("[WARN] no auth providers defined")
}
return providers
}
// resetEnv clears all sensitive env vars
func resetEnv(envs ...string) {
for _, env := range envs {
if err := os.Unsetenv(env); err != nil {
log.Printf("[WARN] can't unset env %s, %s", env, err)
if _, err := p.Parse(); err != nil {
if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
os.Exit(0)
} else {
os.Exit(1)
}
}
}
backend/app/main_test.go
+24 -144
View File
@@ -1,168 +1,48 @@
package main
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestApplication(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18080
return o
})
func TestMain(t *testing.T) {
go func() { _ = app.Run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18080/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
// add comment
resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
require.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
body, _ = ioutil.ReadAll(resp.Body)
t.Log(string(body))
assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
app.Wait()
}
func TestApplicationDevMode(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18085
o.DevPasswd = "password"
o.Auth.Dev = true
return o
})
go func() { _ = app.Run(ctx) }()
time.Sleep(100 * time.Millisecond) // let server start
assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers), "extra auth provider")
assert.Equal(t, "dev", app.restSrv.Authenticator.Providers[4].Name, "dev auth provider")
// send ping
resp, err := http.Get("http://localhost:18085/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
app.Wait()
}
func TestApplicationFailed(t *testing.T) {
opts := Opts{}
p := flags.NewParser(&opts, flags.Default)
// RO bolt location
_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp",
"--store.bolt.path=/dev/null"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
"open /dev/null/remark.db: not a directory")
t.Log(err)
// RO backup location
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
"--backup=/dev/null/not-writable"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
t.Log(err)
// invalid url
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "--backup=/tmp", "----store.bolt.path=/tmp"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
t.Log(err)
opts = Opts{}
_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=mongo"})
assert.Nil(t, err)
_, err = New(opts)
assert.EqualError(t, err, "unsupported store type mongo")
t.Log(err)
}
func TestApplicationShutdown(t *testing.T) {
app, ctx := prepApp(t, 500*time.Millisecond, func(o Opts) Opts {
o.Port = 18090
return o
})
st := time.Now()
err := app.Run(ctx)
assert.Nil(t, err)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
app.Wait()
}
func TestApplicationMainSignal(t *testing.T) {
os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
"--port=18100", "--url=https://demo.remark42.com"}
os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg"}
go func() {
time.Sleep(100 * time.Millisecond)
time.Sleep(500 * time.Millisecond)
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
require.Nil(t, err)
}()
st := time.Now()
main()
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
}
func prepApp(t *testing.T, duration time.Duration, fn func(o Opts) Opts) (*Application, context.Context) {
opts := Opts{}
// prepare options
p := flags.NewParser(&opts, flags.Default)
_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
require.Nil(t, err)
opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", opts.Port)
opts.Store.Bolt.Timeout = 10 * time.Second
opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
opts.BackupLocation = "/tmp"
opts = fn(opts)
os.Remove(opts.Store.Bolt.Path + "/remark.db")
// create app
app, err := New(opts)
require.Nil(t, err)
ctx, cancel := context.WithCancel(context.Background())
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
time.Sleep(duration)
log.Print("[TEST] terminate app")
cancel()
st := time.Now()
main()
assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
wg.Done()
}()
return app, ctx
time.Sleep(50 * time.Millisecond) // let server start
// send ping
resp, err := http.Get("http://localhost:18202/api/v1/ping")
require.Nil(t, err)
defer resp.Body.Close()
assert.Equal(t, 200, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, "pong", string(body))
wg.Wait()
}
backend/app/migrator/disqus.go
+5 -5
View File
@@ -68,14 +68,14 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
}
if failed > 0 {
return passed, errors.Errorf("failed to save %d comments", failed)
err = errors.Errorf("failed to save %d comments", failed)
if passed == 0 {
err = errors.New("import failed")
}
}
log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
if failed > 0 && passed == 0 {
err = errors.New("import failed")
}
return passed, err
}
@@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
return commentsCh
}
func (d *Disqus) cleanText(text string) string {
func (*Disqus) cleanText(text string) string {
text = strings.Replace(text, "\n", "", -1)
text = strings.Replace(text, "\t", "", -1)
return text
backend/app/migrator/disqus_test.go
+6 -5
View File
@@ -9,6 +9,7 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
@@ -19,9 +20,9 @@ func TestDisqus_Import(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := service.DataStore{Interface: b}
dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
d := Disqus{DataStore: &dataStore}
size, err := d.Import(strings.NewReader(xmlTest), "test")
size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
assert.Nil(t, err)
assert.Equal(t, 3, size)
@@ -36,7 +37,7 @@ func TestDisqus_Import(t *testing.T) {
assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator)
assert.Equal(t, "Dmitry Noname", c.User.Name)
assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID)
assert.Equal(t, "96243f024cf6ad42b66f0c72709ae20b5d10ec14", c.User.IP)
assert.Equal(t, "7001968ea3f6c9013a9f0a3650f200c10c927638", c.User.IP)
posts, err := dataStore.List("test", 0, 0)
assert.Nil(t, err)
@@ -49,7 +50,7 @@ func TestDisqus_Import(t *testing.T) {
func TestDisqus_Convert(t *testing.T) {
d := Disqus{}
ch := d.convert(strings.NewReader(xmlTest), "test")
ch := d.convert(strings.NewReader(xmlTestDisqus), "test")
res := []store.Comment{}
for comment := range ch {
@@ -74,7 +75,7 @@ func TestDisqus_Convert(t *testing.T) {
assert.Equal(t, exp0, res[0])
}
var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
<category dsq:id="707279">
backend/app/migrator/migrator.go
+3
View File
@@ -9,6 +9,7 @@ import (
"os"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
@@ -46,6 +47,8 @@ func ImportComments(p ImportParams) (int, error) {
switch p.Provider {
case "disqus":
importer = &Disqus{DataStore: p.DataStore}
case "wordpress":
importer = &WordPress{DataStore: p.DataStore}
case "native":
importer = &Remark{DataStore: p.DataStore}
default:
backend/app/migrator/migrator_test.go
+32 -5
View File
@@ -8,9 +8,10 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/service"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_ImportDisqus(t *testing.T) {
@@ -19,12 +20,12 @@ func TestMigrator_ImportDisqus(t *testing.T) {
os.Remove("/tmp/disqus-test.xml")
}()
err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600)
err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
require.Nil(t, err)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b}
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/disqus-test.xml",
@@ -39,6 +40,32 @@ func TestMigrator_ImportDisqus(t *testing.T) {
assert.Equal(t, 3, len(last), "3 comments imported")
}
func TestMigrator_ImportWordPress(t *testing.T) {
defer func() {
os.Remove("/tmp/remark-test.db")
os.Remove("/tmp/wordpress-test.xml")
}()
err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
require.Nil(t, err)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/wordpress-test.xml",
SiteID: "test",
Provider: "wordpress",
})
assert.Nil(t, err)
assert.Equal(t, 3, size)
last, err := dataStore.Last("test", 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(last), "3 comments imported")
}
func TestMigrator_ImportRemark(t *testing.T) {
defer func() {
os.Remove("/tmp/remark-test.db")
@@ -53,7 +80,7 @@ func TestMigrator_ImportRemark(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b}
dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
@@ -70,7 +97,7 @@ func TestMigrator_ImportRemark(t *testing.T) {
}
func TestMigrator_ImportFailed(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
dataStore := &service.DataStore{Interface: b}
backend/app/migrator/remark.go
+1
View File
@@ -8,6 +8,7 @@ import (
"log"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
backend/app/migrator/remark_test.go
+11 -4
View File
@@ -10,7 +10,9 @@ import (
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -18,6 +20,7 @@ import (
var testDb = "/tmp/test-remark.db"
func TestRemark_Export(t *testing.T) {
defer os.Remove(testDb)
b := prep(t)
r := Remark{DataStore: b}
@@ -29,11 +32,13 @@ func TestRemark_Export(t *testing.T) {
c1, err := buf.ReadString('\n')
assert.Nil(t, err)
log.Print(c1)
exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
assert.Equal(t, exp, c1)
}
func TestRemark_Import(t *testing.T) {
defer os.Remove(testDb)
r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
@@ -46,7 +51,7 @@ func TestRemark_Import(t *testing.T) {
os.Remove(testDb)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{DataStore: &service.DataStore{Interface: b}}
r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
size, err := r.Import(buf, "radio-t")
assert.Nil(t, err)
assert.Equal(t, 2, size)
@@ -60,6 +65,8 @@ func TestRemark_Import(t *testing.T) {
}
func TestRemark_ImportManyWithError(t *testing.T) {
defer os.Remove(testDb)
goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
buf := &bytes.Buffer{}
@@ -72,7 +79,7 @@ func TestRemark_ImportManyWithError(t *testing.T) {
os.Remove(testDb)
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
r := Remark{DataStore: &service.DataStore{Interface: b}}
r := Remark{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
n, err := r.Import(buf, "radio-t")
assert.EqualError(t, err, "failed to save 2 comments")
assert.Equal(t, 1200, n)
@@ -88,7 +95,7 @@ func prep(t *testing.T) *service.DataStore {
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
b := &service.DataStore{Interface: boltStore}
b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
comment := store.Comment{
ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
backend/app/migrator/wordpress.go
+156
View File
@@ -0,0 +1,156 @@
package migrator
import (
"encoding/xml"
"html"
"io"
"log"
"time"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
const wpTimeLayout = "2006-01-02 15:04:05"
// WordPress implements Importer from WP xml
type WordPress struct {
DataStore Store
}
type wpItem struct {
Link string `xml:"link"`
Comments []wpComment `xml:"comment"`
}
type wpComment struct {
ID string `xml:"comment_id"`
Author string `xml:"comment_author"`
AuthorEmail string `xml:"comment_author_email"`
AuthorIP string `xml:"comment_author_IP"`
Date wpTime `xml:"comment_date_gmt"`
Content string `xml:"comment_content"`
Approved string `xml:"comment_approved"`
PID string `xml:"comment_parent"`
}
type wpTime struct {
time time.Time
}
func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var v string
if err := d.DecodeElement(&v, &start); err != nil {
return err
}
t, err := time.Parse(wpTimeLayout, v)
if err != nil {
return err
}
w.time = t
return err
}
// Convert satisfies formatter.CommentConverter
func (w *WordPress) Convert(text string) string {
return html.UnescapeString(text) // sanitize remains on comment create
}
// Import comments from WP and save to store
func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
if err = w.DataStore.DeleteAll(siteID); err != nil {
return 0, err
}
commentsCh := w.convert(r, siteID)
failed, passed := 0, 0
for c := range commentsCh {
if _, err = w.DataStore.Create(c); err != nil {
failed++
continue
}
passed++
}
if failed > 0 {
err = errors.Errorf("failed to save %d comments", failed)
if passed == 0 {
err = errors.New("import failed")
}
}
log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
return passed, err
}
func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
decoder := xml.NewDecoder(r)
commentsCh := make(chan store.Comment)
stats := struct {
inpItems, failedItems int
inpComments, failedComments int
rejectedComments int // not approved
}{}
commentFormatter := store.NewCommentFormatter(w)
go func() {
for {
t, err := decoder.Token()
if t == nil || err != nil {
break
}
switch el := t.(type) {
case xml.StartElement:
if el.Name.Local == "item" {
stats.inpItems++
item := wpItem{}
if err := decoder.DecodeElement(&item, &el); err != nil {
log.Printf("[WARN] Can't decode item, %s", err)
stats.failedItems++
continue
}
if item.Comments != nil {
for _, comment := range item.Comments {
if comment.Approved != "1" {
stats.rejectedComments++
continue
}
if comment.PID == "0" {
comment.PID = ""
}
c := store.Comment{
ID: comment.ID,
Locator: store.Locator{URL: item.Link, SiteID: siteID},
User: store.User{
ID: "wordpress_" + store.EncodeID(comment.Author),
Name: comment.Author,
IP: comment.AuthorIP,
},
Text: comment.Content,
Timestamp: comment.Date.time,
ParentID: comment.PID,
}
commentsCh <- commentFormatter.Format(c)
stats.inpComments++
if stats.inpComments%1000 == 0 {
log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
}
}
}
}
}
}
close(commentsCh)
log.Printf("[INFO] converted %d comments, %+v", stats.inpComments-stats.failedComments, stats)
}()
return commentsCh
}
backend/app/migrator/wordpress_test.go
+322
View File
@@ -0,0 +1,322 @@
package migrator
import (
"os"
"strings"
"testing"
"time"
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestWordPress_Import(t *testing.T) {
siteID := "testWP"
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
assert.Nil(t, err, "create store")
dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
wp := WordPress{DataStore: &dataStore}
size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
assert.Nil(t, err)
assert.Equal(t, 3, size)
last, err := dataStore.Last(siteID, 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(last), "3 comments imported")
c := last[0]
assert.Equal(t, "14", c.ID)
assert.Equal(t, store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}, c.Locator)
assert.Equal(t, "wordpress_75b2b81081f82495d7af26759e67af6554ffda4a", c.User.ID)
assert.Equal(t, "SuperUser3", c.User.Name)
assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
assert.Equal(t, ts, c.Timestamp)
assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
posts, err := dataStore.List(siteID, 0, 0)
assert.Nil(t, err)
assert.Equal(t, 1, len(posts))
p := posts[0]
assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
assert.Nil(t, err)
assert.Equal(t, 3, count)
}
func TestWordPress_Convert(t *testing.T) {
wp := WordPress{}
ch := wp.convert(strings.NewReader(xmlTestWP), "testWP")
comments := []store.Comment{}
for c := range ch {
comments = append(comments, c)
}
assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
exp1 := store.Comment{
ID: "13",
Locator: store.Locator{
SiteID: "testWP",
URL: "https://realmenweardress.es/2010/07/do-you-rp/",
},
Text: `<p>[…] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. […]</p>` + "\n",
User: store.User{
Name: "Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual",
ID: "wordpress_" + store.EncodeID("Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual"),
IP: "74.200.244.101",
},
}
exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
assert.Equal(t, exp1, comments[1])
}
func TestWP_Convert_MD(t *testing.T) {
wp := WordPress{}
ch := wp.convert(strings.NewReader(xmlTestWPmd), "siteID")
comments := []store.Comment{}
for c := range ch {
comments = append(comments, c)
}
assert.Equal(t, 3, len(comments), "3 comments exported")
assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
assert.Equal(t, "<p>markdown <code>text</code></p>\n", comments[1].Text)
expText := `<p>Row1 Link <a href="http://releases.rancher.com/os/latest">http://releases.rancher.com/os/latest</a> markdown <code>text</code> blah</p>`
expText += "\n\n<p>Row3 markdown<code>md block</code></p>\n"
assert.Equal(t, expText, comments[2].Text)
}
var xmlTestWP = `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0"
xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/"
xmlns:content="http://purl.org/rss/1.0/modules/content/"
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:wp="http://wordpress.org/export/1.2/"
>
<channel>
<title>Real Men Wear Dress.es</title>
<link>https://realmenweardress.es</link>
<description>SuperAdmin&#039;s gaming and technological musings</description>
<pubDate>Mon, 23 Jul 2018 10:21:47 +0000</pubDate>
<language>en-US</language>
<wp:wxr_version>1.2</wp:wxr_version>
<wp:base_site_url>https://realmenweardress.es</wp:base_site_url>
<wp:base_blog_url>https://realmenweardress.es</wp:base_blog_url>
<wp:author><wp:author_id>2</wp:author_id><wp:author_login><![CDATA[SuperAdmin]]></wp:author_login><wp:author_email><![CDATA[superadmin@super.eu]]></wp:author_email><wp:author_display_name><![CDATA[SuperAdmin]]></wp:author_display_name><wp:author_first_name><![CDATA[SuperAdmin]]></wp:author_first_name><wp:author_last_name><![CDATA[superadmin]]></wp:author_last_name></wp:author>
<wp:author><wp:author_id>1</wp:author_id><wp:author_login><![CDATA[admin]]></wp:author_login><wp:author_email><![CDATA[superadmin@superadmin.co.uk]]></wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
<wp:category>
<wp:term_id>25</wp:term_id>
<wp:category_nicename><![CDATA[cataclysm]]></wp:category_nicename>
<wp:category_parent><![CDATA[]]></wp:category_parent>
<wp:cat_name><![CDATA[Cataclysm]]></wp:cat_name>
</wp:category>
<wp:tag>
<wp:term_id>39</wp:term_id>
<wp:tag_slug><![CDATA[addons]]></wp:tag_slug>
<wp:tag_name><![CDATA[addons]]></wp:tag_name>
</wp:tag>
<generator>https://wordpress.org/?v=4.8.1</generator>
<item>
<title>Post without comments</title>
<link>https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/</link>
<pubDate>Sat, 19 Jun 2010 08:34:13 +0000</pubDate>
<dc:creator><![CDATA[admin]]></dc:creator>
<guid isPermaLink="false">http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg</guid>
<description></description>
<content:encoded><![CDATA[So you can actually fly into the well it appears and if your lucky you stay mounted. I imagine it terrifies the poor rats.]]></content:encoded>
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
<wp:post_id>6</wp:post_id>
<wp:post_date><![CDATA[2010-06-19 08:34:13]]></wp:post_date>
<wp:post_date_gmt><![CDATA[2010-06-19 08:34:13]]></wp:post_date_gmt>
<wp:comment_status><![CDATA[open]]></wp:comment_status>
<wp:ping_status><![CDATA[open]]></wp:ping_status>
<wp:post_name><![CDATA[screenshot_013110_200413]]></wp:post_name>
<wp:status><![CDATA[inherit]]></wp:status>
<wp:post_parent>1</wp:post_parent>
<wp:menu_order>0</wp:menu_order>
<wp:post_type><![CDATA[attachment]]></wp:post_type>
<wp:post_password><![CDATA[]]></wp:post_password>
<wp:is_sticky>0</wp:is_sticky>
<wp:attachment_url><![CDATA[https://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:attachment_url>
<wp:postmeta>
<wp:meta_key><![CDATA[_wp_attached_file]]></wp:meta_key>
<wp:meta_value><![CDATA[2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:meta_value>
</wp:postmeta>
</item>
<item>
<title>Post with comments. One is not approved</title>
<link>https://realmenweardress.es/2010/07/do-you-rp/</link>
<pubDate>Mon, 19 Jul 2010 14:24:22 +0000</pubDate>
<dc:creator><![CDATA[SuperAdmin]]></dc:creator>
<guid isPermaLink="false">http://realmenweardress.es/?p=100</guid>
<description></description>
<content:encoded><![CDATA[<a href="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546180886.jpeg"><img class="size-thumbnail wp-image-102 alignleft" title="I need to stand on things else I can't reach" src="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546270587-120x120.jpg" alt="I need to stand on things else I can't reach" width="120" height="120" /></a>Meet Grokknomel?]]></content:encoded>
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
<wp:post_id>100</wp:post_id>
<wp:post_date><![CDATA[2010-07-19 14:24:22]]></wp:post_date>
<wp:post_date_gmt><![CDATA[2010-07-19 14:24:22]]></wp:post_date_gmt>
<wp:comment_status><![CDATA[open]]></wp:comment_status>
<wp:ping_status><![CDATA[open]]></wp:ping_status>
<wp:post_name><![CDATA[do-you-rp]]></wp:post_name>
<wp:status><![CDATA[publish]]></wp:status>
<wp:post_parent>0</wp:post_parent>
<wp:menu_order>0</wp:menu_order>
<wp:post_type><![CDATA[post]]></wp:post_type>
<wp:post_password><![CDATA[]]></wp:post_password>
<wp:is_sticky>0</wp:is_sticky>
<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
<wp:postmeta>
<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
<wp:meta_value><![CDATA[2]]></wp:meta_value>
</wp:postmeta>
<wp:comment>
<wp:comment_id>8</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser1]]></wp:comment_author>
<wp:comment_author_email><![CDATA[superuser1@aol.com]]></wp:comment_author_email>
<wp:comment_author_url>http://superuser1.blogspot.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[79.141.141.73]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-20 12:08:08]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-20 12:08:08]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[I do catch myself]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>9</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser2]]></wp:comment_author>
<wp:comment_author_email><![CDATA[superuser2@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>http://thewowstorm.wordpress.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[97.36.113.1]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-20 13:09:25]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-20 13:09:25]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[I think it us inherent in the game to start seeing your character as a personality]]></wp:comment_content>
<wp:comment_approved><![CDATA[0]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>13</wp:comment_id>
<wp:comment_author><![CDATA[Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual]]></wp:comment_author>
<wp:comment_author_email><![CDATA[]]></wp:comment_author_email>
<wp:comment_author_url>http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[74.200.244.101]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-21 14:02:08]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-21 14:02:08]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[[...] I know I&#8217;m a bit loony with my attachment to my bankers.  I&#8217;m glad I&#8217;m not the only one. [...]]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[pingback]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>14</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser3]]></wp:comment_author>
<wp:comment_author_email><![CDATA[blablah@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>http://realmenweardress.es</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>13</wp:comment_parent>
<wp:comment_user_id>2</wp:comment_user_id>
</wp:comment>
</item>
</channel>
</rss>
`
// parts of unused xml tags are omitted
var xmlTestWPmd = `
<?xml version="1.0" encoding="UTF-8" ?>
<channel>
<item>
<title>Deploying RancherOS on Vultr instances</title>
<link>https://realmenweardress.es/2016/07/deploying-rancheros-on-vultr-instances/</link>
<wp:comment>
<wp:comment_id>1</wp:comment_id>
<wp:comment_author><![CDATA[user1]]></wp:comment_author>
<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>https://eric.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[Row1
Row2
Row4]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>2</wp:comment_id>
<wp:comment_author><![CDATA[user1]]></wp:comment_author>
<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>https://eric.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[markdown ` + "`" + "text" + "`" + `]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>2</wp:comment_id>
<wp:comment_author><![CDATA[user1]]></wp:comment_author>
<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>https://eric.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[Row1 Link http://releases.rancher.com/os/latest markdown ` + "`" + "text" + "`" + ` blah
Row3 markdown` +
"```" +
"md block" +
"```" +
`]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
</item>
</channel>
</rss>
`
backend/app/rest/api/admin.go
+29 -39
View File
@@ -1,21 +1,19 @@
package api
import (
"compress/gzip"
"errors"
"fmt"
"io"
"log"
"net/http"
"path"
"time"
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -23,10 +21,11 @@ import (
// admin provides router for all requests available for admin users only
type admin struct {
dataService *service.DataStore
exporter migrator.Exporter
cache cache.LoadingCache
authenticator auth.Authenticator
readOnlyAge int
avatarProxy *proxy.Avatar
migrator *Migrator
}
func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
@@ -38,10 +37,12 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
router.Get("/user/{userid}", a.getUserInfoCtrl)
router.Get("/deleteme", a.deleteMeRequestCtrl)
router.Put("/verify/{userid}", a.setVerifyCtrl)
router.Get("/export", a.exportCtrl)
router.Put("/pin/{id}", a.setPinCtrl)
router.Get("/blocked", a.blockedUsersCtrl)
router.Put("/readonly", a.setReadOnlyCtrl)
a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
return router
}
@@ -57,7 +58,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
return
}
a.cache.Flush(locator.SiteID, locator.URL)
a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"id": id, "locator": locator})
}
@@ -73,7 +74,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
return
}
a.cache.Flush(siteID, userID)
a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID))
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
}
@@ -108,11 +109,25 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID)
// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
if !claims.Flags.DeleteMe {
rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
return
}
if err := a.dataService.DeleteUser(claims.SiteID, claims.User.ID); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
return
}
a.cache.Flush(claims.SiteID, claims.User.ID)
if claims.User.Picture != "" {
if err := a.avatarProxy.Store.Remove(path.Base(claims.User.Picture)); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
return
}
}
a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, lastCommentsScope))
render.Status(r, http.StatusOK)
render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
}
@@ -134,7 +149,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
return
}
a.cache.Flush(siteID, userID)
a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID))
render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
}
@@ -171,7 +186,7 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
return
}
a.cache.Flush(locator.SiteID)
a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus})
}
@@ -185,7 +200,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
return
}
a.cache.Flush(siteID, userID)
a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus})
}
@@ -200,35 +215,10 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
return
}
a.cache.Flush(locator.URL)
a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus})
}
// GET /export?site=site-id?mode=file|stream
// exports all comments for siteID as json stream or gz file
func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
var writer io.Writer = w
if r.URL.Query().Get("mode") == "file" {
exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
w.WriteHeader(http.StatusOK)
gzWriter := gzip.NewWriter(w)
defer func() {
if e := gzWriter.Close(); e != nil {
log.Printf("[WARN] can't close gzip writer, %s", e)
}
}()
writer = gzWriter
}
if _, err := a.exporter.Export(writer, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
}
func (a *admin) checkBlocked(siteID string, user store.User) bool {
return a.dataService.IsBlocked(siteID, user.ID)
}
backend/app/rest/api/admin_test.go
+42 -18
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"time"
@@ -13,6 +14,7 @@ import (
"github.com/dgrijalva/jwt-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/store"
)
@@ -20,7 +22,7 @@ import (
func TestAdmin_Delete(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -51,7 +53,7 @@ func TestAdmin_Delete(t *testing.T) {
func TestAdmin_DeleteUser(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -105,7 +107,7 @@ func TestAdmin_DeleteUser(t *testing.T) {
func TestAdmin_Pin(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -149,7 +151,7 @@ func TestAdmin_Pin(t *testing.T) {
func TestAdmin_Block(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -204,7 +206,7 @@ func TestAdmin_Block(t *testing.T) {
assert.Equal(t, false, j["block"])
// block with ttl
code, _ = block(1, "10ms")
code, _ = block(1, "50ms")
require.Equal(t, 200, code)
res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
@@ -216,7 +218,7 @@ func TestAdmin_Block(t *testing.T) {
assert.Equal(t, "", comments.Comments[0].Text)
assert.True(t, comments.Comments[0].Deleted)
time.Sleep(11 * time.Millisecond)
time.Sleep(50 * time.Millisecond)
res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
assert.Equal(t, 200, code)
comments = commentsWithInfo{}
@@ -230,7 +232,7 @@ func TestAdmin_Block(t *testing.T) {
func TestAdmin_BlockedList(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
client := http.Client{}
@@ -272,7 +274,7 @@ func TestAdmin_BlockedList(t *testing.T) {
func TestAdmin_ReadOnly(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -318,7 +320,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
func TestAdmin_ReadOnlyWithAge(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"},
@@ -360,7 +362,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
func TestAdmin_Verify(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -411,13 +413,12 @@ func TestAdmin_Verify(t *testing.T) {
assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
assert.Equal(t, "test test #1", comments.Comments[0].Text)
assert.False(t, comments.Comments[0].User.Verified)
}
func TestAdmin_ExportStream(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -437,7 +438,7 @@ func TestAdmin_ExportStream(t *testing.T) {
func TestAdmin_ExportFile(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -469,7 +470,7 @@ func TestAdmin_ExportFile(t *testing.T) {
func TestAdmin_DeleteMeRequest(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -495,9 +496,15 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
},
User: &store.User{
ID: "user1",
ID: "user1",
Picture: "pic.image",
},
}
claims.Flags.DeleteMe = true
_ = os.MkdirAll("/tmp/42", 0700)
defer os.RemoveAll("/tmp/42")
ioutil.WriteFile("/tmp/42/pic.image", []byte("some image data"), 0600)
token, err := srv.Authenticator.JWTService.Token(&claims)
assert.Nil(t, err)
@@ -517,7 +524,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -552,6 +559,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
ID: "user1",
},
}
claims.Flags.DeleteMe = true
token, err := srv.Authenticator.JWTService.Token(&claims)
assert.Nil(t, err)
@@ -573,12 +581,27 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
resp, err = client.Do(req)
assert.Nil(t, err)
assert.Equal(t, 400, resp.StatusCode, resp.Status)
// try without deleteme flag
badClaims2 := claims
badClaims2.Flags.DeleteMe = false
token, err = srv.Authenticator.JWTService.Token(&badClaims2)
assert.Nil(t, err)
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
assert.Nil(t, err)
req.SetBasicAuth("dev", "password")
resp, err = client.Do(req)
assert.Nil(t, err)
assert.Equal(t, 403, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.True(t, strings.Contains(string(b), "can't use provided token"))
}
func TestAdmin_GetUserInfo(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -595,7 +618,8 @@ func TestAdmin_GetUserInfo(t *testing.T) {
u := store.User{}
err = json.Unmarshal([]byte(body), &u)
assert.Nil(t, err)
assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "", Admin: false, Blocked: false, Verified: false}, u)
assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "823688dafca7393d24c871a2da98a84d8732e927",
Admin: false, Blocked: false, Verified: false}, u)
_, code = get(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL))
assert.Equal(t, 401, code, "no auth")
backend/app/rest/api/middleware.go
+22
View File
@@ -14,6 +14,7 @@ import (
"time"
"github.com/go-chi/chi/middleware"
"github.com/umputun/remark/backend/app/rest"
)
@@ -105,6 +106,7 @@ func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler)
if qun, err := url.QueryUnescape(q); err == nil {
q = qun
}
q = sanitizeQuery(q)
remoteIP := strings.Split(r.RemoteAddr, ":")[0]
if strings.HasPrefix(r.RemoteAddr, "[") {
@@ -158,6 +160,26 @@ func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user stri
return body, user
}
func sanitizeQuery(u string) string {
out := []rune(u)
hide := []string{"password", "passwd", "secret", "credentials"}
for _, h := range hide {
if strings.Contains(strings.ToLower(u), h+"=") {
stPos := strings.Index(strings.ToLower(u), h+"=") + len(h) + 1
fnPos := strings.Index(u[stPos:], "&")
if fnPos == -1 {
fnPos = len(u)
} else {
fnPos = stPos + fnPos
}
for i := stPos; i < fnPos; i++ {
out[i] = rune('*')
}
}
}
return string(out)
}
func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool {
for _, flg := range flags {
if (flg == LogAll && f != LogNone) || flg == f {
backend/app/rest/api/middleware_test.go
+24 -4
View File
@@ -9,10 +9,10 @@ import (
"github.com/go-chi/chi"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/stretchr/testify/require"
)
func TestMiddleware_AppInfo(t *testing.T) {
@@ -39,13 +39,17 @@ func TestMiddleware_AppInfo(t *testing.T) {
}
func TestMiddleware_GetBodyAndUser(t *testing.T) {
req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body"))
req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body1\nbody2"))
require.Nil(t, err)
body, user := getBodyAndUser(req, []LoggerFlag{LogAll})
assert.Equal(t, "body", body)
assert.Equal(t, "body1 body2", body)
assert.Equal(t, "", user, "no user")
b, err := ioutil.ReadAll(req.Body)
assert.NoError(t, err)
assert.Equal(t, "body1\nbody2", string(b))
req = rest.SetUserInfo(req, store.User{ID: "id1", Name: "user1"})
_, user = getBodyAndUser(req, []LoggerFlag{LogAll})
assert.Equal(t, ` - id1 "user1"`, user, "no user")
@@ -62,3 +66,19 @@ func TestMiddleware_GetBodyAndUser(t *testing.T) {
assert.Equal(t, "", body)
assert.Equal(t, ` - id1 "user1"`, user, "no user")
}
func TestMiddleware_sanitizeReqURL(t *testing.T) {
tbl := []struct {
in string
out string
}{
{"", ""},
{"/aa/bb?xyz=123", "/aa/bb?xyz=123"},
{"/aa/bb?xyz=123&secret=asdfghjk", "/aa/bb?xyz=123&secret=********"},
{"/aa/bb?xyz=123&secret=asdfghjk&key=val", "/aa/bb?xyz=123&secret=********&key=val"},
{"/aa/bb?xyz=123&secret=asdfghjk&key=val&password=1234", "/aa/bb?xyz=123&secret=********&key=val&password=****"},
}
for i, tt := range tbl {
assert.Equal(t, tt.out, sanitizeQuery(tt.in), "check #%d, %s", i, tt.in)
}
}
backend/app/rest/api/migrator.go
+41 -83
View File
@@ -2,18 +2,13 @@ package api
import (
"compress/gzip"
"context"
"fmt"
"io"
"log"
"net/http"
"strings"
"sync"
"time"
"github.com/didip/tollbooth"
"github.com/didip/tollbooth_chi"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/render"
"github.com/umputun/remark/backend/app/migrator"
@@ -21,114 +16,77 @@ import (
"github.com/umputun/remark/backend/app/rest/cache"
)
// Migrator rest runs on unexposed port and available for local requests only
// Migrator rest with import and export controllers
type Migrator struct {
Version string
Cache cache.LoadingCache
NativeImporter migrator.Importer
DisqusImporter migrator.Importer
NativeExported migrator.Exporter
SecretKey string
httpServer *http.Server
lock sync.Mutex
Cache cache.LoadingCache
NativeImporter migrator.Importer
DisqusImporter migrator.Importer
WordPressImporter migrator.Importer
NativeExported migrator.Exporter
KeyStore KeyStore
}
// Run the listener and request's router, activate rest server
// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
func (m *Migrator) Run(port int) {
log.Printf("[INFO] activate import server on port %d", port)
router := m.routes()
m.lock.Lock()
m.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
m.lock.Unlock()
err := m.httpServer.ListenAndServe()
log.Printf("[WARN] http server terminated, %s", err)
// KeyStore defines sub-interface for consumers needed just a key
type KeyStore interface {
Key(siteID string) (key string, err error)
}
// Shutdown import http server
func (m *Migrator) Shutdown() {
log.Print("[WARN] shutdown import server")
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
m.lock.Lock()
if m.httpServer != nil {
if err := m.httpServer.Shutdown(ctx); err != nil {
log.Printf("[DEBUG] importer shutdown error, %s", err)
}
}
m.lock.Unlock()
log.Print("[DEBUG] shutdown import server completed")
}
func (m *Migrator) routes() chi.Router {
router := chi.NewRouter()
router.Use(middleware.RealIP, Recoverer)
router.Use(middleware.Throttle(1000), middleware.Timeout(15*time.Minute))
router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
router.Use(AppInfo("remark42-migrator", m.Version), Ping, Logger(nil, LogAll))
router.Post("/api/v1/admin/import", m.importCtrl)
router.Get("/api/v1/admin/export", m.exportCtrl)
func (m *Migrator) withRoutes(router chi.Router) chi.Router {
router.Get("/export", m.exportCtrl)
router.Post("/import", m.importCtrl)
return router
}
// POST /import?secret=key&site=site-id&provider=disqus|remark
// POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
// imports comments from post body.
func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
secret := r.URL.Query().Get("secret")
if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
render.Status(r, http.StatusForbidden)
render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
return
}
siteID := r.URL.Query().Get("site")
importer := m.NativeImporter
if r.URL.Query().Get("provider") == "disqus" {
var importer migrator.Importer
switch r.URL.Query().Get("provider") {
case "disqus":
importer = m.DisqusImporter
case "wordpress":
importer = m.WordPressImporter
default:
importer = m.NativeImporter
}
log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, r.URL.Query().Get("provider"))
size, err := importer.Import(r.Body, siteID)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
return
}
m.Cache.Flush(siteID)
m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
render.Status(r, http.StatusCreated)
render.JSON(w, r, JSON{"status": "ok", "size": size})
}
// GET /export?site=site-id&secret=12345
// GET /export?site=site-id&secret=12345&?mode=file|stream
// exports all comments for siteID as gz file
func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
secret := r.URL.Query().Get("secret")
if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
render.Status(r, http.StatusForbidden)
render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
return
}
siteID := r.URL.Query().Get("site")
exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
w.WriteHeader(http.StatusOK)
gzWriter := gzip.NewWriter(w)
defer func() {
if e := gzWriter.Close(); e != nil {
log.Printf("[WARN] can't close gzip writer, %s", e)
}
}()
var writer io.Writer = w
if r.URL.Query().Get("mode") == "file" {
exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
w.WriteHeader(http.StatusOK)
gzWriter := gzip.NewWriter(w)
defer func() {
if e := gzWriter.Close(); e != nil {
log.Printf("[WARN] can't close gzip writer, %s", e)
}
}()
writer = gzWriter
}
if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil {
if _, err := m.NativeExported.Export(writer, siteID); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
return
}
backend/app/rest/api/migrator_test.go
+248 -32
View File
@@ -2,6 +2,7 @@ package api
import (
"compress/gzip"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -11,16 +12,21 @@ import (
"time"
"github.com/coreos/bbolt"
"github.com/go-chi/chi"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
func TestMigrator_Import(t *testing.T) {
srv, ts := prepImportSrv(t)
srv, _, ts := prepImportSrv(t)
assert.NotNil(t, srv)
defer cleanupImportSrv(srv, ts)
@@ -28,7 +34,7 @@ func TestMigrator_Import(t *testing.T) {
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r)
assert.Nil(t, err)
resp, err := client.Do(req)
assert.Nil(t, err)
@@ -39,8 +45,44 @@ func TestMigrator_Import(t *testing.T) {
assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b))
}
func TestMigrator_ImportFromWP(t *testing.T) {
srv, ds, ts := prepImportSrv(t)
assert.NotNil(t, srv)
defer cleanupImportSrv(srv, ts)
r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress&secret=123456", r)
assert.Nil(t, err)
req.Header.Add("Content-Type", "application/xml; charset=utf-8")
resp, err := client.Do(req)
assert.Nil(t, err)
assert.Equal(t, http.StatusCreated, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, `{"size":3,"status":"ok"}`+"\n", string(b))
assert.NoError(t, ds.Interface.Close())
srvAccess, tsAccess := prep(t)
require.NotNil(t, srvAccess)
defer cleanup(ts, srvAccess)
res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
require.Equal(t, 200, code)
comments := []store.Comment{}
err = json.Unmarshal([]byte(res), &comments)
assert.Nil(t, err)
assert.Equal(t, 3, len(comments), "should have 3 comments")
t.Logf("%+v", comments)
assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
comments[0].Text)
}
func TestMigrator_ImportRejected(t *testing.T) {
srv, ts := prepImportSrv(t)
srv, _, ts := prepImportSrv(t)
assert.NotNil(t, srv)
defer cleanupImportSrv(srv, ts)
@@ -48,15 +90,15 @@ func TestMigrator_ImportRejected(t *testing.T) {
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
assert.Nil(t, err)
resp, err := client.Do(req)
assert.Nil(t, err)
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
func TestMigrator_Export(t *testing.T) {
srv, ts := prepImportSrv(t)
srv, _, ts := prepImportSrv(t)
assert.NotNil(t, srv)
defer cleanupImportSrv(srv, ts)
@@ -64,13 +106,14 @@ func TestMigrator_Export(t *testing.T) {
{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
client := &http.Client{Timeout: 1 * time.Second}
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=123456", r)
require.Nil(t, err)
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=123456", nil)
// check file mode
req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t&secret=123456", nil)
require.Nil(t, err)
resp, err = client.Do(req)
require.Nil(t, err)
@@ -85,42 +128,215 @@ func TestMigrator_Export(t *testing.T) {
assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
t.Logf("%s", string(ungzBody))
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=bad", nil)
// check stream mode
req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t&secret=123456", nil)
require.Nil(t, err)
resp, err = client.Do(req)
require.Nil(t, err)
require.Equal(t, 403, resp.StatusCode)
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
body, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
assert.Equal(t, 2, strings.Count(string(body), "\n"))
assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
t.Logf("%s", string(body))
req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t&secret=bad", nil)
require.Nil(t, err)
resp, err = client.Do(req)
require.Nil(t, err)
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
func TestMigrator_Shutdown(t *testing.T) {
srv := Migrator{}
go func() {
time.Sleep(100 * time.Millisecond)
srv.Shutdown()
}()
st := time.Now()
srv.Run(0)
assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
}
func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) {
func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
dataStore := &service.DataStore{Interface: b}
adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
svc = &Migrator{
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
NativeExported: &migrator.Remark{DataStore: dataStore},
Cache: &mockCache{},
SecretKey: "123456",
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
NativeExported: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
KeyStore: adminStore,
}
routes := svc.routes()
a := auth.Authenticator{
DevPasswd: "password",
Providers: nil,
KeyStore: adminStore,
JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour),
}
routes := svc.withRoutes(chi.NewRouter().With(a.Auth(true)).With(a.AdminOnly))
ts = httptest.NewServer(routes)
return svc, ts
return svc, dataStore, ts
}
func cleanupImportSrv(srv *Migrator, ts *httptest.Server) {
func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
ts.Close()
os.Remove(testDb)
}
var xmlTestWP = `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0"
xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/"
xmlns:content="http://purl.org/rss/1.0/modules/content/"
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:wp="http://wordpress.org/export/1.2/"
>
<channel>
<title>Real Men Wear Dress.es</title>
<link>https://realmenweardress.es</link>
<description>SuperAdmin&#039;s gaming and technological musings</description>
<pubDate>Mon, 23 Jul 2018 10:21:47 +0000</pubDate>
<language>en-US</language>
<wp:wxr_version>1.2</wp:wxr_version>
<wp:base_site_url>https://realmenweardress.es</wp:base_site_url>
<wp:base_blog_url>https://realmenweardress.es</wp:base_blog_url>
<wp:author><wp:author_id>2</wp:author_id><wp:author_login><![CDATA[SuperAdmin]]></wp:author_login><wp:author_email><![CDATA[superadmin@super.eu]]></wp:author_email><wp:author_display_name><![CDATA[SuperAdmin]]></wp:author_display_name><wp:author_first_name><![CDATA[SuperAdmin]]></wp:author_first_name><wp:author_last_name><![CDATA[superadmin]]></wp:author_last_name></wp:author>
<wp:author><wp:author_id>1</wp:author_id><wp:author_login><![CDATA[admin]]></wp:author_login><wp:author_email><![CDATA[superadmin@superadmin.co.uk]]></wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
<wp:category>
<wp:term_id>25</wp:term_id>
<wp:category_nicename><![CDATA[cataclysm]]></wp:category_nicename>
<wp:category_parent><![CDATA[]]></wp:category_parent>
<wp:cat_name><![CDATA[Cataclysm]]></wp:cat_name>
</wp:category>
<wp:tag>
<wp:term_id>39</wp:term_id>
<wp:tag_slug><![CDATA[addons]]></wp:tag_slug>
<wp:tag_name><![CDATA[addons]]></wp:tag_name>
</wp:tag>
<generator>https://wordpress.org/?v=4.8.1</generator>
<item>
<title>Post without comments</title>
<link>https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/</link>
<pubDate>Sat, 19 Jun 2010 08:34:13 +0000</pubDate>
<dc:creator><![CDATA[admin]]></dc:creator>
<guid isPermaLink="false">http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg</guid>
<description></description>
<content:encoded><![CDATA[So you can actually fly into the well it appears and if your lucky you stay mounted. I imagine it terrifies the poor rats.]]></content:encoded>
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
<wp:post_id>6</wp:post_id>
<wp:post_date><![CDATA[2010-06-19 08:34:13]]></wp:post_date>
<wp:post_date_gmt><![CDATA[2010-06-19 08:34:13]]></wp:post_date_gmt>
<wp:comment_status><![CDATA[open]]></wp:comment_status>
<wp:ping_status><![CDATA[open]]></wp:ping_status>
<wp:post_name><![CDATA[screenshot_013110_200413]]></wp:post_name>
<wp:status><![CDATA[inherit]]></wp:status>
<wp:post_parent>1</wp:post_parent>
<wp:menu_order>0</wp:menu_order>
<wp:post_type><![CDATA[attachment]]></wp:post_type>
<wp:post_password><![CDATA[]]></wp:post_password>
<wp:is_sticky>0</wp:is_sticky>
<wp:attachment_url><![CDATA[https://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:attachment_url>
<wp:postmeta>
<wp:meta_key><![CDATA[_wp_attached_file]]></wp:meta_key>
<wp:meta_value><![CDATA[2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:meta_value>
</wp:postmeta>
</item>
<item>
<title>Post with comments. One is not approved</title>
<link>https://realmenweardress.es/2010/07/do-you-rp/</link>
<pubDate>Mon, 19 Jul 2010 14:24:22 +0000</pubDate>
<dc:creator><![CDATA[SuperAdmin]]></dc:creator>
<guid isPermaLink="false">http://realmenweardress.es/?p=100</guid>
<description></description>
<content:encoded><![CDATA[<a href="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546180886.jpeg"><img class="size-thumbnail wp-image-102 alignleft" title="I need to stand on things else I can't reach" src="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546270587-120x120.jpg" alt="I need to stand on things else I can't reach" width="120" height="120" /></a>Meet Grokknomel?]]></content:encoded>
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
<wp:post_id>100</wp:post_id>
<wp:post_date><![CDATA[2010-07-19 14:24:22]]></wp:post_date>
<wp:post_date_gmt><![CDATA[2010-07-19 14:24:22]]></wp:post_date_gmt>
<wp:comment_status><![CDATA[open]]></wp:comment_status>
<wp:ping_status><![CDATA[open]]></wp:ping_status>
<wp:post_name><![CDATA[do-you-rp]]></wp:post_name>
<wp:status><![CDATA[publish]]></wp:status>
<wp:post_parent>0</wp:post_parent>
<wp:menu_order>0</wp:menu_order>
<wp:post_type><![CDATA[post]]></wp:post_type>
<wp:post_password><![CDATA[]]></wp:post_password>
<wp:is_sticky>0</wp:is_sticky>
<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
<wp:postmeta>
<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
<wp:meta_value><![CDATA[2]]></wp:meta_value>
</wp:postmeta>
<wp:comment>
<wp:comment_id>8</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser1]]></wp:comment_author>
<wp:comment_author_email><![CDATA[superuser1@aol.com]]></wp:comment_author_email>
<wp:comment_author_url>http://superuser1.blogspot.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[79.141.141.73]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-20 12:08:08]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-20 12:08:08]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[I do catch myself]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>9</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser2]]></wp:comment_author>
<wp:comment_author_email><![CDATA[superuser2@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>http://thewowstorm.wordpress.com</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[97.36.113.1]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-20 13:09:25]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-20 13:09:25]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[I think it us inherent in the game to start seeing your character as a personality]]></wp:comment_content>
<wp:comment_approved><![CDATA[0]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>13</wp:comment_id>
<wp:comment_author><![CDATA[Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual]]></wp:comment_author>
<wp:comment_author_email><![CDATA[]]></wp:comment_author_email>
<wp:comment_author_url>http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[74.200.244.101]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-07-21 14:02:08]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-07-21 14:02:08]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[[...] I know I&#8217;m a bit loony with my attachment to my bankers.  I&#8217;m glad I&#8217;m not the only one. [...]]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[pingback]]></wp:comment_type>
<wp:comment_parent>0</wp:comment_parent>
<wp:comment_user_id>0</wp:comment_user_id>
</wp:comment>
<wp:comment>
<wp:comment_id>14</wp:comment_id>
<wp:comment_author><![CDATA[SuperUser3]]></wp:comment_author>
<wp:comment_author_email><![CDATA[blablah@gmail.com]]></wp:comment_author_email>
<wp:comment_author_url>http://realmenweardress.es</wp:comment_author_url>
<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
<wp:comment_content><![CDATA[Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this 'base-url' are failing.
I switched to Github with success:
'''
set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1
'''
Thanks for the article!]]></wp:comment_content>
<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
<wp:comment_type><![CDATA[]]></wp:comment_type>
<wp:comment_parent>13</wp:comment_parent>
<wp:comment_user_id>2</wp:comment_user_id>
</wp:comment>
</item>
</channel>
</rss>
`
backend/app/rest/api/rest.go
+45 -23
View File
@@ -17,11 +17,11 @@ import (
"github.com/didip/tollbooth_chi"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/cors"
"github.com/go-chi/render"
"github.com/pkg/errors"
"gopkg.in/russross/blackfriday.v2"
"github.com/rakyll/statik/fs"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
@@ -32,16 +32,20 @@ import (
// Rest is a rest access server
type Rest struct {
Version string
DataService *service.DataStore
Authenticator auth.Authenticator
Exporter migrator.Exporter
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
Version string
DataService *service.DataStore
Authenticator auth.Authenticator
Cache cache.LoadingCache
AvatarProxy *proxy.Avatar
ImageProxy *proxy.Image
CommentFormatter *store.CommentFormatter
Migrator *Migrator
WebRoot string
RemarkURL string
ReadOnlyAge int
SharedSecret string
ScoreThresholds struct {
Low int
Critical int
@@ -55,9 +59,7 @@ type Rest struct {
const hardBodyLimit = 1024 * 64 // limit size of body
var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
blackfriday.BackslashLineBreak | blackfriday.Autolink
const lastCommentsScope = "last"
type commentsWithInfo struct {
Comments []store.Comment `json:"comments"`
@@ -68,10 +70,6 @@ type commentsWithInfo struct {
func (s *Rest) Run(port int) {
log.Printf("[INFO] activate rest server on port %d", port)
if s.DataService != nil && len(s.DataService.Admins) > 0 {
log.Printf("[DEBUG] admins %+v", s.DataService.Admins)
}
router := s.routes()
s.lock.Lock()
@@ -111,13 +109,24 @@ func (s *Rest) routes() chi.Router {
s.adminService = admin{
dataService: s.DataService,
exporter: s.Exporter,
migrator: s.Migrator,
cache: s.Cache,
authenticator: s.Authenticator,
readOnlyAge: s.ReadOnlyAge,
avatarProxy: s.AvatarProxy,
}
ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization
corsMiddleware := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
ExposedHeaders: []string{"Authorization"},
AllowCredentials: true,
MaxAge: 300,
})
router.Use(corsMiddleware.Handler)
ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
// auth routes for all providers
router.Route("/auth", func(r chi.Router) {
@@ -172,7 +181,7 @@ func (s *Rest) routes() chi.Router {
rauth.Post("/deleteme", s.deleteMeCtrl)
// admin routes, admin users only
rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll)))
rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly))
})
})
@@ -202,11 +211,24 @@ func (s *Rest) routes() chi.Router {
return router
}
// serves static files from /web
// serves static files from /web or embedded by statik
func addFileServer(r chi.Router, path string, root http.FileSystem) {
log.Printf("[INFO] run file server for %s, path %s", root, path)
var webFS http.Handler
statikFS, err := fs.New()
if err == nil {
log.Printf("[INFO] run file server for %s, embedded", root)
webFS = http.FileServer(statikFS)
}
if err != nil {
log.Printf("[DEBUG] no embedded assets loaded, %s", err)
log.Printf("[INFO] run file server for %s, path %s", root, path)
webFS = http.FileServer(root)
}
origPath := path
fs := http.StripPrefix(path, http.FileServer(root))
webFS = http.StripPrefix(path, webFS)
if path != "/" && path[len(path)-1] != '/' {
r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
path += "/"
@@ -220,7 +242,7 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
http.NotFound(w, r)
return
}
fs.ServeHTTP(w, r)
webFS.ServeHTTP(w, r)
}))
}
backend/app/rest/api/rest_private.go
+21 -43
View File
@@ -14,10 +14,10 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/hashicorp/go-multierror"
"gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -31,11 +31,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
user, err := rest.GetUserInfo(r)
if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
log.Printf("[DEBUG] create comment %+v", comment)
comment.PrepareUntrusted() // clean all fields user not supposed to set
@@ -43,12 +39,12 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
comment.Orig = comment.Text // original comment text, prior to md render
if err = s.DataService.ValidateComment(&comment); err != nil {
if err := s.DataService.ValidateComment(&comment); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
return
}
comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
comment.Text = s.ImageProxy.Convert(comment.Text)
comment = s.CommentFormatter.Format(comment)
// check if user blocked
if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
@@ -74,7 +70,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
return
}
s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)
s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
render.Status(r, http.StatusCreated)
render.JSON(w, r, &finalComment)
@@ -86,6 +83,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
edit := struct {
Text string
Summary string
Delete bool
}{}
if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
@@ -93,17 +91,14 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
user, err := rest.GetUserInfo(r)
if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
id := chi.URLParam(r, "id")
log.Printf("[DEBUG] update comment %s", id)
var currComment store.Comment
var err error
if currComment, err = s.DataService.Get(locator, id); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
return
@@ -114,12 +109,11 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
text = s.ImageProxy.Convert(text)
editReq := service.EditRequest{
Text: text,
Text: s.CommentFormatter.FormatText(edit.Text),
Orig: edit.Text,
Summary: edit.Summary,
Delete: edit.Delete,
}
res, err := s.DataService.EditComment(locator, id, editReq)
@@ -128,18 +122,13 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
s.Cache.Flush(locator.URL, "last", user.ID)
s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID))
render.JSON(w, r, res)
}
// GET /user?site=siteID - returns user info
func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
user, err := rest.GetUserInfo(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
if siteID := r.URL.Query().Get("site"); siteID != "" {
user.Verified = s.DataService.IsVerified(siteID, user.ID)
}
@@ -149,12 +138,7 @@ func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
// PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
user, err := rest.GetUserInfo(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
id := chi.URLParam(r, "id")
log.Printf("[DEBUG] vote for comment %s", id)
@@ -172,18 +156,14 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
return
}
s.Cache.Flush(locator.URL, comment.User.ID)
s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
}
// GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
user, err := rest.GetUserInfo(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
userB, err := json.Marshal(&user)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
@@ -241,11 +221,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
// POST /deleteme?site_id=site - requesting delete of all user info
// makes jwt with user info and sends it back as a part of json response
func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
user, err := rest.GetUserInfo(r)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
return
}
user := rest.MustGetUserInfo(r)
siteID := r.URL.Query().Get("site")
claims := auth.CustomClaims{
@@ -257,12 +233,14 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
},
User: &user,
}
claims.Flags.DeleteMe = true // prevent this token from being used for login
tokenStr, err := s.Authenticator.JWTService.Token(&claims)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
return
}
link := fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", s.RemarkURL, tokenStr)
link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
}
backend/app/rest/api/rest_private_test.go
+57 -17
View File
@@ -19,15 +19,15 @@ import (
func TestRest_Create(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
resp, err := post(t, ts.URL+"/api/v1/comment",
`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
assert.Nil(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
c := JSON{}
err = json.Unmarshal(b, &c)
assert.Nil(t, err)
@@ -40,7 +40,7 @@ func TestRest_Create(t *testing.T) {
func TestRest_CreateOldPost(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
// make old, but not too old comment
old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
@@ -74,7 +74,7 @@ func TestRest_CreateOldPost(t *testing.T) {
func TestRest_CreateTooBig(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
@@ -106,7 +106,7 @@ func TestRest_CreateRejected(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`
// try to create without auth
@@ -114,14 +114,15 @@ func TestRest_CreateRejected(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 401, resp.StatusCode)
}
func TestRest_CreateAndGet(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
// create comment
resp, err := post(t, ts.URL+"/api/v1/comment",
`{"text": "**test** *123* http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
`{"text": "**test** *123*\n\n http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
require.Nil(t, err)
require.Equal(t, http.StatusCreated, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
@@ -138,8 +139,8 @@ func TestRest_CreateAndGet(t *testing.T) {
comment := store.Comment{}
err = json.Unmarshal([]byte(res), &comment)
assert.Nil(t, err)
assert.Equal(t, `<p><strong>test</strong> <em>123</em> <a href="http://radio-t.com" rel="nofollow">http://radio-t.com</a></p>`+"\n", comment.Text)
assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig)
assert.Equal(t, "<p><strong>test</strong> <em>123</em></p>\n\n<p><a href=\"http://radio-t.com\" rel=\"nofollow\">http://radio-t.com</a></p>\n", comment.Text)
assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig)
assert.Equal(t, store.User{Name: "developer one", ID: "dev",
Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
comment.User)
@@ -149,7 +150,7 @@ func TestRest_CreateAndGet(t *testing.T) {
func TestRest_Update(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "p1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -185,10 +186,49 @@ func TestRest_Update(t *testing.T) {
assert.Equal(t, c2, c3, "same as response from update")
}
func TestRest_UpdateDelete(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "p1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
id := addComment(t, c1, ts)
client := http.Client{}
req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
assert.Nil(t, err)
req.SetBasicAuth("dev", "password")
b, err := client.Do(req)
assert.Nil(t, err)
body, err := ioutil.ReadAll(b.Body)
assert.Nil(t, err)
assert.Equal(t, 200, b.StatusCode, string(body))
// comments returned by update
c2 := store.Comment{}
err = json.Unmarshal(body, &c2)
assert.Nil(t, err)
assert.Equal(t, id, c2.ID)
assert.True(t, c2.Deleted)
// read updated comment
res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
assert.Equal(t, 200, code)
c3 := store.Comment{}
err = json.Unmarshal([]byte(res), &c3)
assert.Nil(t, err)
assert.Equal(t, "", c3.Text)
assert.Equal(t, "", c3.Orig)
assert.True(t, c3.Deleted)
}
func TestRest_UpdateNotOwner(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "p1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
@@ -220,7 +260,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
func TestRest_Vote(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -264,7 +304,7 @@ func TestRest_Vote(t *testing.T) {
func TestRest_UserAllData(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
// write 3 comments
user := store.User{ID: "dev", Name: "user name 1"}
@@ -319,7 +359,7 @@ func TestRest_UserAllData(t *testing.T) {
func TestRest_UserAllDataManyComments(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
user := store.User{ID: "dev", Name: "user name 1"}
c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
@@ -353,7 +393,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
func TestRest_DeleteMe(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
client := http.Client{}
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
@@ -375,7 +415,7 @@ func TestRest_DeleteMe(t *testing.T) {
claims, err := srv.Authenticator.JWTService.Parse(token)
assert.Nil(t, err)
assert.Equal(t, "dev", claims.User.ID)
assert.Equal(t, "https://demo.remark42.com/api/v1/admin/deleteme?token="+token, m["link"])
assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"])
req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
assert.Nil(t, err)
backend/app/rest/api/rest_public.go
+20 -25
View File
@@ -10,7 +10,6 @@ import (
"github.com/go-chi/chi"
"github.com/go-chi/render"
"gopkg.in/russross/blackfriday.v2"
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/rest/cache"
@@ -27,7 +26,8 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
}
log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Find(locator, sort)
if e != nil {
return nil, e
@@ -78,10 +78,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
return
}
//comment.Text = string(blackfriday.Run([]byte(comment.Text),
// blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
comment.Text = s.ImageProxy.Convert(comment.Text)
comment = s.CommentFormatter.Format(comment)
comment.Sanitize()
render.HTML(w, r, comment.Text)
}
@@ -90,7 +87,8 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL)
data, err := s.Cache.Get(key, func() ([]byte, error) {
info, e := s.DataService.Info(locator, s.ReadOnlyAge)
if e != nil {
return nil, e
@@ -116,22 +114,15 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
limit = 0
}
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), func() ([]byte, error) {
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, limit)
if e != nil {
return nil, e
}
comments = s.adminService.alterComments(comments, r)
// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
filterDeleted := []store.Comment{}
for _, c := range comments {
if c.Deleted {
continue
}
filterDeleted = append(filterDeleted, c)
}
filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
return encodeJSONWithHTML(filterDeleted)
})
@@ -179,7 +170,8 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), func() ([]byte, error) {
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(userID, siteID)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.User(siteID, userID, limit, 0)
if e != nil {
return nil, e
@@ -203,6 +195,8 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
// GET /config?site=siteID - returns configuration
func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
type config struct {
Version string `json:"version"`
EditDuration int `json:"edit_duration"`
@@ -219,8 +213,8 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
Version: s.Version,
EditDuration: int(s.DataService.EditDuration.Seconds()),
MaxCommentSize: s.DataService.MaxCommentSize,
Admins: s.DataService.Admins,
AdminEmail: s.Authenticator.AdminEmail,
Admins: s.DataService.AdminStore.Admins(siteID),
AdminEmail: s.DataService.AdminStore.Email(siteID),
LowScore: s.ScoreThresholds.Low,
CriticalScore: s.ScoreThresholds.Critical,
ReadOnlyAge: s.ReadOnlyAge,
@@ -259,15 +253,15 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
}
// key could be long for multiple posts, make it sha1
key := cache.URLKey(r) + strings.Join(posts, ",")
k := cache.URLKey(r) + strings.Join(posts, ",")
hasher := sha1.New()
if _, err := hasher.Write([]byte(key)); err != nil {
if _, err := hasher.Write([]byte(k)); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
return
}
sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
data, err := s.Cache.Get(cache.Key(sha, siteID), func() ([]byte, error) {
key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
data, err := s.Cache.Get(key, func() ([]byte, error) {
counts, e := s.DataService.Counts(siteID, posts)
if e != nil {
return nil, e
@@ -295,7 +289,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
skip = v
}
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), func() ([]byte, error) {
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID)
data, err := s.Cache.Get(key, func() ([]byte, error) {
posts, e := s.DataService.List(siteID, limit, skip)
if e != nil {
return nil, e
backend/app/rest/api/rest_public_test.go
+16 -15
View File
@@ -19,7 +19,7 @@ import (
func TestRest_Ping(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
res, code := get(t, ts.URL+"/api/v1/ping")
assert.Equal(t, "pong", res)
@@ -29,7 +29,7 @@ func TestRest_Ping(t *testing.T) {
func TestRest_Preview(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
assert.Nil(t, err)
@@ -42,7 +42,7 @@ func TestRest_Preview(t *testing.T) {
func TestRest_PreviewWithMD(t *testing.T) {
srv, ts := prep(t)
require.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
text := `
# h1
@@ -70,7 +70,7 @@ BKT
func TestRest_Find(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
assert.Equal(t, 400, code, "nothing in")
@@ -124,7 +124,7 @@ func TestRest_Find(t *testing.T) {
func TestRest_FindAge(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
@@ -156,7 +156,7 @@ func TestRest_FindAge(t *testing.T) {
func TestRest_FindReadOnly(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
@@ -198,7 +198,7 @@ func TestRest_FindReadOnly(t *testing.T) {
func TestRest_Last(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1", ParentID: "p1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -238,12 +238,13 @@ func TestRest_Last(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
assert.Nil(t, err)
assert.Equal(t, 2, len(comments), "should have 2 comments")
t.Logf("%+v", comments)
}
func TestRest_FindUserComments(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -280,7 +281,7 @@ func TestRest_FindUserComments(t *testing.T) {
func TestRest_UserInfo(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
body, code := getWithAuth(t, ts.URL+"/api/v1/user?site=radio-t")
assert.Equal(t, 200, code)
@@ -294,7 +295,7 @@ func TestRest_UserInfo(t *testing.T) {
func TestRest_Count(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -324,7 +325,7 @@ func TestRest_Count(t *testing.T) {
func TestRest_Counts(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -354,7 +355,7 @@ func TestRest_Counts(t *testing.T) {
func TestRest_List(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
c1 := store.Comment{Text: "test test #1",
Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -381,7 +382,7 @@ func TestRest_List(t *testing.T) {
func TestRest_Config(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
assert.Equal(t, 200, code)
@@ -401,7 +402,7 @@ func TestRest_Config(t *testing.T) {
func TestRest_Info(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
@@ -439,7 +440,7 @@ func TestRest_Info(t *testing.T) {
func TestRest_Robots(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
body, code := get(t, ts.URL+"/robots.txt")
assert.Equal(t, 200, code)
backend/app/rest/api/rest_test.go
+46 -26
View File
@@ -17,8 +17,11 @@ import (
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
adminstore "github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -30,7 +33,7 @@ var getStartedHTML = "/tmp/getstarted.html"
func TestRest_FileServer(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
body, code := get(t, ts.URL+"/web/test-remark.html")
assert.Equal(t, 200, code)
@@ -40,7 +43,7 @@ func TestRest_FileServer(t *testing.T) {
func TestRest_GetStarted(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700)
assert.Nil(t, err)
@@ -56,7 +59,7 @@ func TestRest_GetStarted(t *testing.T) {
}
func TestRest_Shutdown(t *testing.T) {
srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300),
srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300),
RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
go func() {
@@ -69,32 +72,56 @@ func TestRest_Shutdown(t *testing.T) {
assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
}
func TestRest_filterComments(t *testing.T) {
user := store.User{ID: "user1", Name: "user name 1"}
c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool {
return c.Text == "test test #1" || c.Text == "test test #3"
})
assert.Equal(t, 2, len(r), "one comment filtered")
}
func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
require.Nil(t, err)
adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
dataStore := &service.DataStore{
Interface: b,
EditDuration: 5 * time.Minute,
MaxCommentSize: 4000,
Secret: "123456",
Admins: []string{"a1", "a2"},
AdminStore: adminStore,
}
srv = &Rest{
DataService: dataStore,
Authenticator: auth.Authenticator{
DevPasswd: "password",
Providers: nil,
AdminEmail: "admin@remark-42.com",
JWTService: auth.NewJWT("12345", false, time.Minute, time.Hour),
DevPasswd: "password",
Providers: nil,
KeyStore: adminStore,
JWTService: auth.NewJWT(adminStore, false, time.Minute, time.Hour),
},
Cache: &cache.Nop{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
Migrator: &Migrator{
DisqusImporter: &migrator.Disqus{DataStore: dataStore},
WordPressImporter: &migrator.WordPress{DataStore: dataStore},
NativeImporter: &migrator.Remark{DataStore: dataStore},
NativeExported: &migrator.Remark{DataStore: dataStore},
Cache: &cache.Nop{},
KeyStore: adminStore,
},
Exporter: &migrator.Remark{DataStore: dataStore},
Cache: &mockCache{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300), RoutePath: "/api/v1/avatar"},
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
}
srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
@@ -156,16 +183,9 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
return crResp["id"].(string)
}
func cleanup(ts *httptest.Server) {
func cleanup(ts *httptest.Server, srv *Rest) {
ts.Close()
srv.DataService.Close()
os.Remove(testDb)
os.Remove(testHTML)
}
type mockCache struct{}
func (mc *mockCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
return fn()
}
func (mc *mockCache) Flush(scopes ...string) {}
backend/app/rest/api/rss.go
+6 -3
View File
@@ -35,7 +35,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
log.Printf("[DEBUG] get rss for post %+v", locator)
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
key := cache.NewKey(locator.SiteID).ID(cache.URLKey(r)).Scopes(locator.SiteID, locator.URL)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Find(locator, "-time")
if e != nil {
return nil, e
@@ -66,7 +67,8 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get rss for site %s", siteID)
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() ([]byte, error) {
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() ([]byte, error) {
comments, e := s.DataService.Last(siteID, maxRssItems)
if e != nil {
return nil, e
@@ -98,7 +100,8 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() (res []byte, e error) {
key := cache.NewKey(siteID).ID(cache.URLKey(r)).Scopes(siteID, lastCommentsScope)
data, err := s.Cache.Get(key, func() (res []byte, e error) {
comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
if e != nil {
return nil, errors.Wrap(e, "can't get last comments")
backend/app/rest/api/rss_test.go
+4 -4
View File
@@ -14,7 +14,7 @@ import (
func TestServer_RssPost(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
waitOnSecChange()
@@ -55,7 +55,7 @@ func TestServer_RssPost(t *testing.T) {
func TestServer_RssSite(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
waitOnSecChange()
@@ -109,7 +109,7 @@ func TestServer_RssSite(t *testing.T) {
func TestServer_RssWithReply(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
waitOnSecChange()
@@ -161,7 +161,7 @@ func TestServer_RssWithReply(t *testing.T) {
func TestServer_RssReplies(t *testing.T) {
srv, ts := prep(t)
assert.NotNil(t, srv)
defer cleanup(ts)
defer cleanup(ts, srv)
waitOnSecChange()
backend/app/rest/auth/auth.go
+64 -16
View File
@@ -15,11 +15,16 @@ import (
type Authenticator struct {
JWTService *JWT
Providers []Provider
AdminEmail string
KeyStore KeyStore
DevPasswd string
PermissionChecker PermissionChecker
}
// KeyStore defines sub-interface for consumers needed just a key
type KeyStore interface {
Key(siteID string) (key string, err error)
}
var devUser = store.User{
ID: "dev",
Name: "developer one",
@@ -27,11 +32,18 @@ var devUser = store.User{
Admin: true,
}
// PermissionChecker defines interface to get user flags
var adminUser = store.User{
ID: "admin",
Name: "admin",
Picture: "/api/v1/avatar/remark.image",
Admin: true,
}
// PermissionChecker defines interface to check user flags
type PermissionChecker interface {
IsVerified(siteID, userID string) bool
IsBlocked(siteID, userID string) bool
IsAdmin(userID string) bool
IsAdmin(siteID, userID string) bool
}
// Auth middleware adds auth from session and populates user info
@@ -40,21 +52,28 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
f := func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
if a.basicDevUser(w, r) { // fail-back to dev user if enabled
user := devUser
r = rest.SetUserInfo(r, user)
// if secret key matches for given site (from request) return admin user
if a.checkSecretKey(r) {
r = rest.SetUserInfo(r, adminUser)
h.ServeHTTP(w, r)
return
}
// use dev user basic auth if enabled
if a.basicDevUser(r) {
r = rest.SetUserInfo(r, devUser)
h.ServeHTTP(w, r)
return
}
claims, err := a.JWTService.Get(r)
if err != nil && reqAuth { // in full auth lack of session causes Unauthorized
log.Printf("[DEBUG] failed auth, %s", err)
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
if err != nil { // in anonymous mode just pass it to the next handler
if err != nil {
if reqAuth { // in full auth lack of token causes Unauthorized
log.Printf("[DEBUG] failed auth, %s", err)
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
// if !reqAuth just pass it to the next handler, used for information only, like logs
h.ServeHTTP(w, r)
return
}
@@ -73,6 +92,12 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
return
}
if a.JWTService.HasFlags(claims) { // flags in token indicate special use cases, not for login
log.Printf("[DEBUG] invalid token flags for %s/%s", claims.User.Name, claims.User.ID)
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
if a.JWTService.IsExpired(claims) {
if claims, err = a.refreshExpiredToken(w, claims); err != nil {
log.Printf("[DEBUG] can't refresh jwt, %s", err)
@@ -90,9 +115,29 @@ func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
return f
}
func (a *Authenticator) checkSecretKey(r *http.Request) bool {
if a.KeyStore == nil {
return false
}
siteID := r.URL.Query().Get("site")
secret := r.URL.Query().Get("secret")
skey, err := a.KeyStore.Key(siteID)
if err != nil {
return false
}
if strings.TrimSpace(secret) == "" || secret != skey {
return false
}
return true
}
// refreshExpiredToken makes new token with passed claims, but only if permission allowed
func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *CustomClaims) (*CustomClaims, error) {
if a.PermissionChecker != nil {
claims.User.Admin = a.PermissionChecker.IsAdmin(claims.User.ID)
claims.User.Admin = a.PermissionChecker.IsAdmin(claims.SiteID, claims.User.ID)
claims.User.Blocked = a.PermissionChecker.IsBlocked(claims.SiteID, claims.User.ID)
claims.User.Verified = a.PermissionChecker.IsVerified(claims.SiteID, claims.User.ID)
}
@@ -103,7 +148,7 @@ func (a *Authenticator) refreshExpiredToken(w http.ResponseWriter, claims *Custo
return claims, nil
}
// AdminOnly allows access to admins
// AdminOnly middleware allows access for admins only
func (a *Authenticator) AdminOnly(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
@@ -123,7 +168,7 @@ func (a *Authenticator) AdminOnly(next http.Handler) http.Handler {
return http.HandlerFunc(fn)
}
func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) bool {
func (a *Authenticator) basicDevUser(r *http.Request) bool {
if a.DevPasswd == "" {
return false
@@ -136,15 +181,18 @@ func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) boo
b, err := base64.StdEncoding.DecodeString(s[1])
if err != nil {
log.Printf("[WARN] dev user auth failed, failed to decode %s, %s", s[1], err)
return false
}
pair := strings.SplitN(string(b), ":", 2)
if len(pair) != 2 {
log.Printf("[WARN] dev user auth failed, failed to split %s", string(b))
return false
}
if pair[0] != "dev" || pair[1] != a.DevPasswd {
log.Printf("[WARN] dev user auth failed, user/passwd mismatch %+v", pair)
return false
}
backend/app/rest/auth/auth_test.go
+54 -3
View File
@@ -11,12 +11,17 @@ import (
"github.com/go-chi/chi"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/admin"
)
var testJwtUserBlocked = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOnRydWV9LCJzdGF0ZSI6IjEyMzQ1NiIsImZyb20iOiJmcm9tIn0.6P_OwGf8CUJRtvNSlW20GmaMb5pFvCNemP94fHCqb5Q"
var testJwtDeleteMe = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZSwiYmxvY2siOmZhbHNlfSwiZmxhZ3MiOnsiZGVsZXRlbWUiOnRydWV9fQ.SLh1QpFytWZqcT99VgcdAOtgFKhvpKCcZwqWTvAd63g"
var testJwtNoUser = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyfQ.sBpblkbBRzZsBSPPNrTWqA5h7h54solrw5L4IypJT_o"
func TestAuthJWTCookie(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour),
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour),
PermissionChecker: &mockUserPermissions{}}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
@@ -51,10 +56,18 @@ func TestAuthJWTCookie(t *testing.T) {
resp, err = client.Do(req)
require.NoError(t, err)
assert.Equal(t, 201, resp.StatusCode, "token expired and refreshed")
req, err = http.NewRequest("GET", server.URL+"/auth", nil)
require.Nil(t, err)
req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtNoUser, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
req.Header.Add("X-XSRF-TOKEN", "random id")
resp, err = client.Do(req)
require.NoError(t, err)
assert.Equal(t, 401, resp.StatusCode, "no user info in the token")
}
func TestAuthJWTHeader(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)}
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
@@ -81,7 +94,7 @@ func TestAuthJWTHeader(t *testing.T) {
}
func TestAuthJWtBlocked(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour, time.Hour)}
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
@@ -100,6 +113,26 @@ func TestAuthJWtBlocked(t *testing.T) {
assert.Equal(t, 401, resp.StatusCode, "blocked user")
}
func TestAuthJWtFlags(t *testing.T) {
a := Authenticator{DevPasswd: "123456", JWTService: NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, time.Hour)}
router := chi.NewRouter()
router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
})
server := httptest.NewServer(router)
defer server.Close()
jar, err := cookiejar.New(nil)
require.Nil(t, err)
client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
req, err := http.NewRequest("GET", server.URL+"/auth", nil)
require.Nil(t, err)
req.Header.Add("X-JWT", testJwtDeleteMe)
resp, err := client.Do(req)
require.NoError(t, err)
assert.Equal(t, 401, resp.StatusCode, "blocked user")
}
func TestAuthRequired(t *testing.T) {
a := Authenticator{DevPasswd: "123456"}
router := chi.NewRouter()
@@ -189,6 +222,24 @@ func TestAdminRequired(t *testing.T) {
}
func TestAuthWithSecret(t *testing.T) {
a := Authenticator{DevPasswd: "123456", KeyStore: admin.NewStaticKeyStore("secretkey")}
router := chi.NewRouter()
router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(201)
})
server := httptest.NewServer(router)
defer server.Close()
resp, err := http.Get(server.URL + "/auth?secret=secretkey")
require.NoError(t, err)
assert.Equal(t, 201, resp.StatusCode, "valid auth user with secret, admin")
resp, err = http.Get(server.URL + "/auth?secret=badsecret")
require.NoError(t, err)
assert.Equal(t, 401, resp.StatusCode, "invalid auth with bad secret")
}
func withBasicAuth(r *http.Request, username, password string) *http.Request {
auth := username + ":" + password
r.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
backend/app/rest/auth/dev_provider.go
+53 -15
View File
@@ -1,6 +1,7 @@
package auth
import (
"bytes"
"context"
"fmt"
"log"
@@ -9,6 +10,8 @@ import (
"sync"
"time"
"github.com/nullrocks/identicon"
"github.com/pkg/errors"
"golang.org/x/oauth2"
"github.com/umputun/remark/backend/app/store"
@@ -18,23 +21,29 @@ const devAuthPort = 8084
// DevAuthServer is a fake oauth server for development
// it provides stand-alone server running on its own port and pretending to be the real oauth2. It also provides
// Dev Provider the same way as normal providers di, i.e. github, google and others.
// can run in interractive and non-interactive mode. In interactive mode login attempts will show login form to select
// desired user name.
// Dev Provider the same way as normal providers do, i.e. like github, google and others.
// can run in interactive and non-interactive mode. In interactive mode login attempts will show login form to select
// desired user name, this is the mode used for development. Non-interactive mode for tests only.
type DevAuthServer struct {
Provider Provider
username string // unsafe, but fine for dev
nonInteractive bool
httpServer *http.Server
lock sync.Mutex
iconGen *identicon.Generator
httpServer *http.Server
lock sync.Mutex
}
// Run oauth2 dev server on port devAuthPort
func (d *DevAuthServer) Run() {
log.Printf("[INFO] run local oauth2 dev server on %d", devAuthPort)
d.lock.Lock()
var err error
d.iconGen, err = identicon.New("github", 5, 3)
if err != nil {
log.Printf("[WARN] can't create identicon, %s", err)
}
d.httpServer = &http.Server{
Addr: fmt.Sprintf(":%d", devAuthPort),
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -42,9 +51,9 @@ func (d *DevAuthServer) Run() {
switch {
case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
// first time it will be called without usernam and will ask for onw
// first time it will be called without username and will ask for one
if !d.nonInteractive && (r.ParseForm() != nil || r.Form.Get("username") == "") {
if _, err := w.Write([]byte(fmt.Sprintf(devUserForm, r.URL.RawQuery))); err != nil {
if _, err = w.Write([]byte(fmt.Sprintf(devUserForm, r.URL.RawQuery))); err != nil {
log.Printf("[WARN] can't write, %s", err)
}
return
@@ -70,19 +79,33 @@ func (d *DevAuthServer) Run() {
"state":"12345678"
}`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if _, err := w.Write([]byte(res)); err != nil {
if _, err = w.Write([]byte(res)); err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
case strings.HasPrefix(r.URL.Path, "/user"):
ava := fmt.Sprintf("http://127.0.0.1:%d/avatar?user=%s", devAuthPort, d.username)
res := fmt.Sprintf(`{
"id": "%s",
"name":"%s"
}`, d.username, d.username)
"name":"%s",
"picture":"%s"
}`, d.username, d.username, ava)
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if _, err := w.Write([]byte(res)); err != nil {
if _, err = w.Write([]byte(res)); err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
case strings.HasPrefix(r.URL.Path, "/avatar"):
user := r.URL.Query().Get("user")
b, e := d.genAvatar(user)
if e != nil {
w.WriteHeader(http.StatusNotFound)
return
}
if _, err = w.Write(b); err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
@@ -94,7 +117,7 @@ func (d *DevAuthServer) Run() {
}
d.lock.Unlock()
err := d.httpServer.ListenAndServe()
err = d.httpServer.ListenAndServe()
log.Printf("[WARN] dev oauth2 server terminated, %s", err)
}
@@ -121,20 +144,35 @@ func NewDev(p Params) Provider {
AuthURL: fmt.Sprintf("http://127.0.0.1:%d/login/oauth/authorize", devAuthPort),
TokenURL: fmt.Sprintf("http://127.0.0.1:%d/login/oauth/access_token", devAuthPort),
},
RedirectURL: "http://127.0.0.1:8080/auth/dev/callback",
RedirectURL: p.RemarkURL + "/auth/dev/callback",
Scopes: []string{"user:email"},
InfoURL: fmt.Sprintf("http://127.0.0.1:%d/user", devAuthPort),
MapUser: func(data userData, _ []byte) store.User {
userInfo := store.User{
ID: data.value("id"),
Name: data.value("name"),
Picture: "",
Picture: data.value("picture"),
}
return userInfo
},
})
}
func (d *DevAuthServer) genAvatar(user string) ([]byte, error) {
if d.iconGen == nil {
return nil, errors.Errorf("no iconGen, skip avatar generation for %s", user)
}
ii, err := d.iconGen.Draw(user) // Generate an IdentIcon
if err != nil {
return nil, errors.Wrapf(err, "failed to draw avatar for %s", user)
}
buf := &bytes.Buffer{}
err = ii.Png(300, buf)
return buf.Bytes(), err
}
var devUserForm = `
<html>
<head>
backend/app/rest/auth/dev_provider_test.go
+12 -3
View File
@@ -13,11 +13,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestDevProvider(t *testing.T) {
params := Params{RemarkURL: "http://127.0.0.1:8080", SecretKey: "123456", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT("12345", false, time.Hour, time.Hour*24*31),
params := Params{RemarkURL: "http://127.0.0.1:8080", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31),
PermissionChecker: &mockUserPermissions{admin: "dev_user"},
}
srv := DevAuthServer{Provider: NewDev(params), nonInteractive: true, username: "dev_user"}
@@ -62,7 +63,15 @@ func TestDevProvider(t *testing.T) {
assert.Nil(t, err)
u := *claims.User
assert.Equal(t, store.User{Name: "dev_user", ID: "dev_user", Picture: "", IP: "",
assert.Equal(t, store.User{Name: "dev_user", ID: "dev_user", Picture: "http://127.0.0.1:8084/avatar?user=dev_user", IP: "",
Admin: true, Blocked: false, Verified: false}, u)
// check avatar
resp, err = client.Get("http://127.0.0.1:8084/avatar?user=dev_user")
require.Nil(t, err)
assert.Equal(t, 200, resp.StatusCode)
body, err = ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, 985, len(body))
t.Logf("headers: %+v", resp.Header)
}
backend/app/rest/auth/jwt.go
+46 -6
View File
@@ -4,7 +4,7 @@ import (
"net/http"
"time"
jwt "github.com/dgrijalva/jwt-go"
"github.com/dgrijalva/jwt-go"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
@@ -13,7 +13,7 @@ import (
// JWT wraps jwt operations
// supports both header and cookie jwt
type JWT struct {
secret string
keyStore KeyStore
secureCookies bool
tokenDuration time.Duration
cookieDuration time.Duration
@@ -29,6 +29,12 @@ type CustomClaims struct {
From string `json:"from,omitempty"`
SiteID string `json:"site_id,omitempty"`
SessionOnly bool `json:"sess_only,omitempty"`
// flags indicate different uses
Flags struct {
Login bool `json:"login,omitempty"`
DeleteMe bool `json:"deleteme,omitempty"`
} `json:"flags,omitempty"`
}
const jwtCookieName = "JWT"
@@ -37,9 +43,9 @@ const xsrfCookieName = "XSRF-TOKEN"
const xsrfHeaderKey = "X-XSRF-TOKEN"
// NewJWT makes JWT service
func NewJWT(secret string, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT {
func NewJWT(keyStore KeyStore, secureCookies bool, tokenDuration time.Duration, cookieDuration time.Duration) *JWT {
res := JWT{
secret: secret,
keyStore: keyStore,
secureCookies: secureCookies,
tokenDuration: tokenDuration,
cookieDuration: cookieDuration,
@@ -50,21 +56,55 @@ func NewJWT(secret string, secureCookies bool, tokenDuration time.Duration, cook
// Token makes jwt with claims
func (j *JWT) Token(claims *CustomClaims) (string, error) {
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
tokenString, err := token.SignedString([]byte(j.secret))
secret, err := j.keyStore.Key(claims.SiteID)
if err != nil {
return "", errors.Wrap(err, "can't get secret")
}
tokenString, err := token.SignedString([]byte(secret))
if err != nil {
return "", errors.Wrap(err, "can't sign jwt token")
}
return tokenString, nil
}
// HasFlags indicates presence of special flags
func (j *JWT) HasFlags(claims *CustomClaims) bool {
return claims.Flags.DeleteMe || claims.Flags.Login
}
// Parse token string and verify. Not checking for expiration
func (j *JWT) Parse(tokenString string) (*CustomClaims, error) {
parser := jwt.Parser{SkipClaimsValidation: true} // allow parsing of expired tokens
getSiteID := func() (siteID string, err error) { // parse token without signature check to get siteID
preToken, _, err := parser.ParseUnverified(tokenString, &CustomClaims{})
if err != nil {
return "", errors.Wrap(err, "can't pre-parse jwt")
}
preClaims, ok := preToken.Claims.(*CustomClaims)
if !ok {
return "", errors.New("invalid jwt")
}
return preClaims.SiteID, nil
}
siteID, err := getSiteID()
if err != nil {
return nil, errors.Wrap(err, "failed to get siteID from jwt token")
}
secret, err := j.keyStore.Key(siteID)
if err != nil {
return nil, errors.Wrap(err, "can't get secret")
}
token, err := parser.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"])
}
return []byte(j.secret), nil
return []byte(secret), nil
})
if err != nil {
return nil, errors.Wrap(err, "can't parse jwt")
backend/app/rest/auth/jwt_test.go
+13 -16
View File
@@ -10,17 +10,14 @@ import (
"github.com/dgrijalva/jwt-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store"
)
var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCI" +
"sImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZS" +
"I6IiIsImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ._loFgh3g45gr9TtGqvM3N584I_6EHEOJnYb6Py84stQ"
var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6InJlb" + "WFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFkbWluIjpmYWxzZX0" + "sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJmbGFncyI6e319.E2Blxqo1wsY855q258c0obxFJ1lgJciv1av1ewzlJBs"
var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6In" +
"JlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsIm" +
"FkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWV9.p6w0sM_NYaRuyhyA9jqfWlB5cx1vZPGhXGC5geSX7nA"
var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIs" + "ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsImFk" + "bWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWUsImZsYWdzIjp7fX0." + "nKhehF1Xiome1yK1ewfOiIsrATvq7Tx7p1BCSJqKHuo"
var testJwtExpired = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MjY4ODc4MjIsImp0aSI6InJhbmRvbSBpZCIs" +
"ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiI" +
@@ -33,7 +30,7 @@ var testJwtBadSign = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4M
var days31 = time.Hour * 24 * 31
func TestJWT_Token(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -56,7 +53,7 @@ func TestJWT_Token(t *testing.T) {
}
func TestJWT_Parse(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims, err := j.Parse(testJwtValid)
assert.NoError(t, err)
assert.False(t, j.IsExpired(claims))
@@ -74,7 +71,7 @@ func TestJWT_Parse(t *testing.T) {
}
func TestJWT_Set(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -89,9 +86,9 @@ func TestJWT_Set(t *testing.T) {
ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
},
SessionOnly: false,
}
claims.SessionOnly = false
rr := httptest.NewRecorder()
err := j.Set(rr, claims, claims.SessionOnly)
assert.Nil(t, err)
@@ -119,7 +116,7 @@ func TestJWT_Set(t *testing.T) {
}
func TestJWT_GetFromHeader(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
req := httptest.NewRequest("GET", "/", nil)
req.Header.Add(jwtHeaderKey, testJwtValid)
@@ -138,13 +135,13 @@ func TestJWT_GetFromHeader(t *testing.T) {
req = httptest.NewRequest("GET", "/", nil)
req.Header.Add(jwtHeaderKey, "bad bad token")
_, err = j.Get(req)
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token contains an invalid number of segments"), err.Error())
require.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "can't pre-parse jwt: token contains an invalid number of segments"), err.Error())
}
func TestJWT_SetAndGetWithCookies(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -186,7 +183,7 @@ func TestJWT_SetAndGetWithCookies(t *testing.T) {
}
func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
@@ -223,7 +220,7 @@ func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
}
func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) {
j := NewJWT("xyz 12345", false, time.Hour, days31)
j := NewJWT(admin.NewStaticKeyStore("xyz 12345"), false, time.Hour, days31)
claims := &CustomClaims{
State: "123456",
backend/app/rest/auth/provider.go
+3 -2
View File
@@ -39,7 +39,6 @@ type Params struct {
AvatarProxy *proxy.Avatar
JwtService *JWT
PermissionChecker PermissionChecker
SecretKey string
Cid string
Csecret string
}
@@ -99,6 +98,7 @@ func (p Provider) loginHandler(w http.ResponseWriter, r *http.Request) {
NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
},
}
claims.Flags.Login = true
if err := p.JwtService.Set(w, &claims, false); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set jwt")
@@ -170,6 +170,7 @@ func (p Provider) authHandler(w http.ResponseWriter, r *http.Request) {
Issuer: "remark42",
Id: p.randToken(),
},
SiteID: oauthClaims.SiteID,
SessionOnly: oauthClaims.SessionOnly,
}
@@ -202,7 +203,7 @@ func (p Provider) setAvatar(u store.User) store.User {
// setPermissions sets permission fields not handled by provider's MapUser, things like admin, verified and blocked
func (p Provider) setPermissions(u store.User, siteID string) store.User {
u.Admin = p.PermissionChecker.IsAdmin(u.ID)
u.Admin = p.PermissionChecker.IsAdmin(siteID, u.ID)
u.Verified = p.PermissionChecker.IsVerified(siteID, u.ID)
u.Blocked = p.PermissionChecker.IsBlocked(siteID, u.ID)
log.Printf("[DEBUG] set permissions for user %s, site %s - %+v", u.ID, siteID, u)
backend/app/rest/auth/provider_test.go
+14 -6
View File
@@ -16,6 +16,7 @@ import (
"golang.org/x/oauth2"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
)
func TestLogin(t *testing.T) {
@@ -52,6 +53,14 @@ func TestLogin(t *testing.T) {
assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser1", Picture: "http://exmple.com/pic1.png",
Admin: false, Blocked: true, IP: ""}, u)
token := resp.Cookies()[0].Value
jwtSvc := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31)
claims, err := jwtSvc.Parse(token)
require.NoError(t, err)
assert.Equal(t, "remark42", claims.Issuer)
assert.Equal(t, "remark", claims.SiteID)
// check admin user
resp, err = client.Get("http://localhost:8981/login?site=remark")
assert.Nil(t, err)
@@ -94,7 +103,7 @@ func TestLoginSessionOnly(t *testing.T) {
req.AddCookie(resp.Cookies()[1])
req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value)
jwtService := NewJWT("12345", false, time.Hour, time.Hour)
jwtService := NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour)
res, err := jwtService.Get(req)
require.Nil(t, err)
assert.Equal(t, true, res.SessionOnly)
@@ -129,13 +138,12 @@ func TestLogout(t *testing.T) {
}
func TestInitProvider(t *testing.T) {
params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret"}
params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret"}
provider := Provider{Name: "test", RedirectURL: "redir"}
res := initProvider(params, provider)
assert.Equal(t, "cid", res.conf.ClientID)
assert.Equal(t, "csecret", res.conf.ClientSecret)
assert.Equal(t, "redir", res.RedirectURL)
assert.Equal(t, "123456", res.SecretKey)
assert.Equal(t, "test", res.Name)
}
@@ -160,8 +168,8 @@ func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Se
},
}
params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT("12345", false, time.Hour, time.Hour*24*31),
params := Params{RemarkURL: "url", Cid: "cid", Csecret: "csecret",
JwtService: NewJWT(admin.NewStaticKeyStore("12345"), false, time.Hour, time.Hour*24*31),
// AvatarProxy: &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"},
PermissionChecker: &mockUserPermissions{admin: "mock_myuser2", verified: "mock_myuser2", blocked: "mock_myuser1"},
}
@@ -225,6 +233,6 @@ type mockUserPermissions struct {
blocked string
}
func (m *mockUserPermissions) IsAdmin(userID string) bool { return userID == m.admin }
func (m *mockUserPermissions) IsAdmin(siteID, userID string) bool { return userID == m.admin }
func (m *mockUserPermissions) IsVerified(siteID, userID string) bool { return userID == m.verified }
func (m *mockUserPermissions) IsBlocked(siteID, userID string) bool { return userID == m.blocked }
backend/app/rest/cache/cache.go
Vendored
+78 -11
View File
@@ -5,32 +5,88 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/rest"
)
// LoadingCache defines interface for caching
type LoadingCache interface {
Get(key string, fn func() ([]byte, error)) (data []byte, err error)
Flush(scopes ...string)
Get(key Key, fn func() ([]byte, error)) (data []byte, err error)
Flush(req FlusherRequest)
}
// Key makes full key from primary key and scopes
func Key(key string, scopes ...string) string {
return strings.Join(scopes, "$$") + "@@" + key
type cacheWithOpts interface {
LoadingCache
setMaxValSize(max int) error
setMaxKeys(max int) error
setMaxCacheSize(max int64) error
setPostFlushFn(postFlushFn func()) error
}
// Key for cache
type Key struct {
id string
siteID string
scopes []string
}
// NewKey makes keys for site
func NewKey(site string) Key {
res := Key{siteID: site}
return res
}
// ID sets key id
func (k Key) ID(id string) Key {
k.id = id
return k
}
// Scopes of the key
func (k Key) Scopes(scopes ...string) Key {
k.scopes = scopes
return k
}
// Merge makes full string key from primary key and scopes
func (k Key) Merge() string {
return strings.Join(k.scopes, "$$") + "@@" + k.id + "@@" + k.siteID
}
// ParseKey gets compound key created by Key func and split it to the actual key and scopes
func ParseKey(fullKey string) (key string, scopes []string, err error) {
func ParseKey(fullKey string) (Key, error) {
elems := strings.Split(fullKey, "@@")
if len(elems) != 2 {
return "", nil, errors.Errorf("can't parse cache key %s", key)
if len(elems) != 3 {
return Key{}, errors.Errorf("can't parse cache key %s", fullKey)
}
scopes = strings.Split(elems[0], "$$")
scopes := strings.Split(elems[0], "$$")
if len(scopes) == 1 && scopes[0] == "" {
scopes = []string{}
}
key = elems[1]
return key, scopes, nil
key := Key{
scopes: scopes,
id: elems[1],
siteID: elems[2],
}
return key, nil
}
// FlusherRequest used as input for cache.Flush
type FlusherRequest struct {
siteID string
scopes []string
}
// Flusher makes new FlusherRequest with empty scopes
func Flusher(siteID string) FlusherRequest {
res := FlusherRequest{siteID: siteID}
return res
}
// Scopes adds scopes to FlusherRequest
func (f FlusherRequest) Scopes(scopes ...string) FlusherRequest {
f.scopes = scopes
return f
}
// URLKey gets url from request to use it as cache key
@@ -43,3 +99,14 @@ func URLKey(r *http.Request) string {
}
return key
}
// Nop does nothing for caching, passing fn call only
type Nop struct{}
// Get calls fn, no actual caching
func (n *Nop) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) {
return fn()
}
// Flush does nothing for NoopCache
func (n *Nop) Flush(req FlusherRequest) {}
backend/app/rest/cache/cache_test.go
Vendored
+10 -9
View File
@@ -15,24 +15,25 @@ func TestCache_Keys(t *testing.T) {
scopes []string
full string
}{
{"key1", []string{"s1"}, "s1@@key1"},
{"key2", []string{"s11", "s2"}, "s11$$s2@@key2"},
{"key3", []string{}, "@@key3"},
{"key1", []string{"s1"}, "s1@@key1@@site"},
{"key2", []string{"s11", "s2"}, "s11$$s2@@key2@@site"},
{"key3", []string{}, "@@key3@@site"},
}
for n, tt := range tbl {
full := Key(tt.key, tt.scopes...)
k := NewKey("site").ID(tt.key).Scopes(tt.scopes...)
full := k.Merge()
assert.Equal(t, tt.full, full, "making key, #%d", n)
k, s, e := ParseKey(full)
k, e := ParseKey(full)
assert.Nil(t, e)
assert.Equal(t, tt.scopes, s)
assert.Equal(t, tt.key, k)
assert.Equal(t, tt.scopes, k.scopes)
assert.Equal(t, tt.key, k.id)
}
_, _, err := ParseKey("abc")
_, err := ParseKey("abc")
assert.Error(t, err)
_, _, err = ParseKey("")
_, err = ParseKey("")
assert.Error(t, err)
}
backend/app/rest/cache/memory.go
Vendored
+40 -8
View File
@@ -20,6 +20,8 @@ type memoryCache struct {
// NewMemoryCache makes memoryCache implementation
func NewMemoryCache(options ...Option) (LoadingCache, error) {
log.Print("[INFO] make memory cache")
res := memoryCache{
postFlushFn: func() {},
maxKeys: 1000,
@@ -48,8 +50,9 @@ func NewMemoryCache(options ...Option) (LoadingCache, error) {
}
// Get is loading cache method to get value by key or load via fn if not found
func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
if b, ok := m.bytesCache.Get(key); ok {
func (m *memoryCache) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) {
mkey := key.Merge()
if b, ok := m.bytesCache.Get(mkey); ok {
return b.([]byte), nil
}
@@ -57,7 +60,7 @@ func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, e
return data, err
}
if m.allowed(data) {
m.bytesCache.Add(key, data)
m.bytesCache.Add(mkey, data)
atomic.AddInt64(&m.currentSize, int64(len(data)))
if m.maxCacheSize > 0 && atomic.LoadInt64(&m.currentSize) > m.maxCacheSize {
@@ -70,9 +73,9 @@ func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, e
}
// Flush clears cache and calls postFlushFn async
func (m *memoryCache) Flush(scopes ...string) {
func (m *memoryCache) Flush(req FlusherRequest) {
if len(scopes) == 0 {
if len(req.scopes) == 0 {
m.bytesCache.Purge()
go m.postFlushFn()
return
@@ -80,12 +83,12 @@ func (m *memoryCache) Flush(scopes ...string) {
// check if fullKey has matching scopes
inScope := func(fullKey string) bool {
_, keyScopes, err := ParseKey(fullKey)
key, err := ParseKey(fullKey)
if err != nil {
return false
}
for _, s := range scopes {
for _, ks := range keyScopes {
for _, s := range req.scopes {
for _, ks := range key.scopes {
if ks == s {
return true
}
@@ -112,3 +115,32 @@ func (m *memoryCache) allowed(data []byte) bool {
}
return true
}
func (m *memoryCache) setMaxValSize(max int) error {
m.maxValueSize = max
if max <= 0 {
return errors.Errorf("negative size for MaxValSize, %d", max)
}
return nil
}
func (m *memoryCache) setMaxKeys(max int) error {
m.maxKeys = max
if max <= 0 {
return errors.Errorf("negative size for MaxKeys, %d", max)
}
return nil
}
func (m *memoryCache) setMaxCacheSize(max int64) error {
m.maxCacheSize = max
if max <= 0 {
return errors.Errorf("negative size or MaxCacheSize, %d", max)
}
return nil
}
func (m *memoryCache) setPostFlushFn(postFlushFn func()) error {
m.postFlushFn = postFlushFn
return nil
}
backend/app/rest/cache/memory_test.go
Vendored
+39 -39
View File
@@ -17,7 +17,7 @@ func TestMemoryCache_Get(t *testing.T) {
var postFnCall, coldCalls int32
lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }))
require.Nil(t, err)
res, err := lc.Get("key", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte("result"), nil
})
@@ -26,7 +26,7 @@ func TestMemoryCache_Get(t *testing.T) {
assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
res, err = lc.Get("key", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte("result"), nil
})
@@ -35,11 +35,11 @@ func TestMemoryCache_Get(t *testing.T) {
assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
lc.Flush()
lc.Flush(Flusher("site"))
time.Sleep(100 * time.Millisecond) // let postFn to do its thing
assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall))
_, err = lc.Get("key", func() ([]byte, error) {
_, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
return nil, errors.New("err")
})
assert.NotNil(t, err)
@@ -53,7 +53,7 @@ func TestMemoryCache_MaxKeys(t *testing.T) {
// put 5 keys to cache
for i := 0; i < 5; i++ {
res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte(fmt.Sprintf("result-%d", i)), nil
})
@@ -64,14 +64,14 @@ func TestMemoryCache_MaxKeys(t *testing.T) {
}
// check if really cached
res, err := lc.Get("key-3", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) {
return []byte("result-blah"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-3", string(res), "should be cached")
// try to cache after maxKeys reached
res, err = lc.Get("key-X", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) {
return []byte("result-X"), nil
})
assert.Nil(t, err)
@@ -80,13 +80,13 @@ func TestMemoryCache_MaxKeys(t *testing.T) {
assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
// put to cache and make sure it cached
res, err = lc.Get("key-Z", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res))
res, err = lc.Get("key-Z", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Zzzz"), nil
})
assert.Nil(t, err)
@@ -98,26 +98,26 @@ func TestMemoryCache_MaxValueSize(t *testing.T) {
lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10))
require.Nil(t, err)
// put good size value to cache and make sure it cached
res, err := lc.Get("key-Z", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res))
res, err = lc.Get("key-Z", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Zzzz"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res), "got cached value")
// put too big value to cache and make sure it is not cached
res, err = lc.Get("key-Big", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) {
return []byte("1234567890"), nil
})
assert.Nil(t, err)
assert.Equal(t, "1234567890", string(res))
res, err = lc.Get("key-Big", func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) {
return []byte("result-big"), nil
})
assert.Nil(t, err)
@@ -129,21 +129,21 @@ func TestMemoryCache_MaxCacheSize(t *testing.T) {
require.Nil(t, err)
// put good size value to cache and make sure it cached
res, err := lc.Get("key-Z", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res))
assert.Equal(t, int64(8), lc.(*memoryCache).currentSize)
_, err = lc.Get("key-Z2", func() ([]byte, error) {
_, err = lc.Get(NewKey("site").ID("key-Z2"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
// this will cause removal
_, err = lc.Get("key-Z3", func() ([]byte, error) {
_, err = lc.Get(NewKey("site").ID("key-Z3"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
@@ -163,7 +163,7 @@ func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) {
go func() {
time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond)
defer wg.Done()
res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) {
return []byte(fmt.Sprintf("result-%d", i)), nil
})
require.Nil(t, err)
@@ -182,7 +182,7 @@ func TestMemoryCache_Parallel(t *testing.T) {
lc, err := NewMemoryCache()
require.Nil(t, err)
res, err := lc.Get("key", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
@@ -194,7 +194,7 @@ func TestMemoryCache_Parallel(t *testing.T) {
i := i
go func() {
defer wg.Done()
res, err := lc.Get("key", func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte(fmt.Sprintf("result-%d", i)), nil
})
@@ -210,28 +210,28 @@ func TestMemoryCache_Scopes(t *testing.T) {
lc, err := NewMemoryCache()
require.Nil(t, err)
res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value", string(res))
res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) {
return []byte("value2"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value2", string(res))
assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
lc.Flush("s1")
lc.Flush(Flusher("site").Scopes("s1"))
assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
_, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
_, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) {
assert.Fail(t, "should stay")
return nil, nil
})
assert.Nil(t, err)
res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte("value-upd"), nil
})
assert.Nil(t, err)
@@ -242,23 +242,23 @@ func TestMemoryCache_Flush(t *testing.T) {
lc, err := NewMemoryCache()
require.Nil(t, err)
addToCache := func(key string, scopes ...string) {
res, err := lc.Get(key, func() ([]byte, error) {
return []byte("value" + key), nil
addToCache := func(id string, scopes ...string) {
res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) {
return []byte("value" + id), nil
})
require.Nil(t, err)
require.Equal(t, "value"+key, string(res))
require.Equal(t, "value"+id, string(res))
}
init := func() {
lc.Flush()
addToCache(Key("key1", "s1", "s2"))
addToCache(Key("key2", "s1", "s2", "s3"))
addToCache(Key("key3", "s1", "s2", "s3"))
addToCache(Key("key4", "s2", "s3"))
addToCache(Key("key5", "s2"))
addToCache(Key("key6"))
addToCache(Key("key7", "s4", "s3"))
lc.Flush(Flusher("site"))
addToCache("key1", "s1", "s2")
addToCache("key2", "s1", "s2", "s3")
addToCache("key3", "s1", "s2", "s3")
addToCache("key4", "s2", "s3")
addToCache("key5", "s2")
addToCache("key6")
addToCache("key7", "s4", "s3")
require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init")
}
@@ -279,7 +279,7 @@ func TestMemoryCache_Flush(t *testing.T) {
for i, tt := range tbl {
init()
lc.Flush(tt.scopes...)
lc.Flush(Flusher("site").Scopes(tt.scopes...))
assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i)
}
}
@@ -287,14 +287,14 @@ func TestMemoryCache_Flush(t *testing.T) {
func TestMemoryCache_FlushFailed(t *testing.T) {
lc, err := NewMemoryCache()
require.Nil(t, err)
val, err := lc.Get("invalid-composite", func() ([]byte, error) {
val, err := lc.Get(NewKey("site").ID("invalid-composite"), func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value", string(val))
assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
lc.Flush("invalid-composite")
lc.Flush(Flusher("site").Scopes("invalid-composite"))
assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
}
backend/app/rest/cache/mongo.go
Vendored
+187
View File
@@ -0,0 +1,187 @@
package cache
import (
"log"
"time"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/go-pkgz/repeater"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
)
type mongoCache struct {
connection *mongo.Connection
postFlushFn func()
maxKeys int
maxValueSize int
maxCacheSize int64
}
const cacheCollection = "cache"
type mongoDoc struct {
SiteID string `bson:"site"`
Key string `bson:"key"`
Scopes []string `bson:"scopes,omitempty"`
Data []byte `bson:"data"`
}
// NewMongoCache makes mongoCache implementation
func NewMongoCache(connection *mongo.Connection, options ...Option) (LoadingCache, error) {
log.Printf("[INFO] make mongo cache with %s", connection)
res := &mongoCache{
connection: connection,
postFlushFn: func() {},
maxKeys: 1000,
maxValueSize: 0,
}
for _, opt := range options {
if err := opt(res); err != nil {
return nil, errors.Wrap(err, "failed to set cache option")
}
}
if err := res.prepare(); err != nil {
return nil, err
}
return res, nil
}
// Get is loading cache method to get value by key or load via fn if not found
func (m *mongoCache) Get(key Key, fn func() ([]byte, error)) (data []byte, err error) {
d := mongoDoc{}
// repeat find from cache with small delay to avoid mgo random error
rep := repeater.NewDefault(5, 10*time.Millisecond)
mgErr := rep.Do(func() error {
return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": key.siteID, "key": key.id}).One(&d)
})
}, mgo.ErrNotFound)
if mgErr == nil { // cached result found
return d.Data, nil
}
if data, err = fn(); err != nil {
return data, err
}
if mgErr != mgo.ErrNotFound { // some other error in mgo query, don't try to update cache
log.Printf("[WARN] unexpected mgo error %+v", mgErr)
return data, err
}
if !m.allowed(data) {
return data, nil
}
d = mongoDoc{
SiteID: key.siteID,
Key: key.id,
Data: data,
Scopes: key.scopes,
}
err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"site": key.siteID, "key": key.id}, bson.M{"$set": d})
return e
})
if err != nil {
return nil, errors.Wrapf(err, "can't set cached value for %+v", key)
}
if m.maxKeys > 0 {
err = m.cleanup(key.siteID)
}
return data, errors.Wrap(err, "failed to cleanup cached records")
}
func (m *mongoCache) cleanup(siteID string) (err error) {
ids := []struct {
ID bson.ObjectId `bson:"_id"`
}{}
err = m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error {
n, countErr := coll.Find(bson.M{"site": siteID}).Count()
if countErr != nil {
return countErr
}
if countErr == nil && n > m.maxKeys {
if findErr := coll.Find(bson.M{"site": siteID}).Sort("+id").Limit(n - m.maxKeys).All(&ids); findErr == nil {
bsonIDs := []bson.ObjectId{}
for _, id := range ids {
bsonIDs = append(bsonIDs, id.ID)
}
_, removalErr := coll.RemoveAll(bson.M{"_id": bson.M{"$in": bsonIDs}})
return removalErr
}
}
return nil
})
return err
}
// Flush clears cache and calls postFlushFn async
func (m *mongoCache) Flush(req FlusherRequest) {
err := m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error {
q := bson.M{"site": req.siteID}
if len(req.scopes) > 0 {
q["scopes"] = bson.M{"$in": req.scopes}
}
_, e := coll.RemoveAll(q)
return e
})
if err == nil && m.postFlushFn != nil {
m.postFlushFn()
}
}
// prepare collections with all indexes
func (m *mongoCache) prepare() error {
errs := new(multierror.Error)
return m.connection.WithCustomCollection(cacheCollection, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "key"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "scopes"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", cacheCollection)
})
}
func (m *mongoCache) allowed(data []byte) bool {
if m.maxValueSize > 0 && len(data) >= m.maxValueSize {
return false
}
return true
}
func (m *mongoCache) setMaxValSize(max int) error {
m.maxValueSize = max
if max <= 0 {
return errors.Errorf("negative size for MaxValSize, %d", max)
}
return nil
}
func (m *mongoCache) setMaxKeys(max int) error {
m.maxKeys = max
if max <= 0 {
return errors.Errorf("negative size for MaxKeys, %d", max)
}
return nil
}
func (m *mongoCache) setMaxCacheSize(max int64) error {
m.maxCacheSize = max
if max <= 0 {
return errors.Errorf("negative size or MaxCacheSize, %d", max)
}
return nil
}
func (m *mongoCache) setPostFlushFn(postFlushFn func()) error {
m.postFlushFn = postFlushFn
return nil
}
backend/app/rest/cache/mongo_test.go
Vendored
+312
View File
@@ -0,0 +1,312 @@
package cache
import (
"fmt"
"log"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMongoCache_Get(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
var postFnCall, coldCalls int32
lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }))
require.Nil(t, err)
res, err := lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte("result"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result", string(res))
assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
res, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte("result"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result", string(res))
assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
lc.Flush(Flusher("site"))
time.Sleep(100 * time.Millisecond) // let postFn to do its thing
assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall))
_, err = lc.Get(NewKey("site").ID("key"), func() ([]byte, error) {
return nil, errors.New("err")
})
assert.NotNil(t, err)
}
func TestMongoCache_MaxKeys(t *testing.T) {
var postFnCall, coldCalls int32
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
lc, err := NewMongoCache(conn, PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }),
MaxKeys(5), MaxValSize(10))
require.Nil(t, err)
// put 5 keys to cache
for i := 0; i < 5; i++ {
res, e := lc.Get(NewKey("site").ID(fmt.Sprintf("key-%d", i)), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte(fmt.Sprintf("result-%d", i)), nil
})
assert.Nil(t, e)
assert.Equal(t, fmt.Sprintf("result-%d", i), string(res))
assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls))
assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
}
// check if really cached
res, err := lc.Get(NewKey("site").ID("key-3"), func() ([]byte, error) {
return []byte("result-blah"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-3", string(res), "should be cached")
// try to cache after maxKeys reached
res, err = lc.Get(NewKey("site").ID("key-X"), func() ([]byte, error) {
return []byte("result-X"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-X", string(res))
conn.WithCustomCollection("cache", func(coll *mgo.Collection) error {
n, e := coll.Find(bson.M{"site": "site"}).Count()
require.NoError(t, e)
require.Equal(t, 5, n)
r := mongoDoc{}
require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r))
assert.Equal(t, "key-1", r.Key)
return nil
})
// put to cache and make sure it cached
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res))
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Zzzz"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res), "got cached value")
conn.WithCustomCollection("cache", func(coll *mgo.Collection) error {
n, e := coll.Find(bson.M{"site": "site"}).Count()
require.NoError(t, e)
require.Equal(t, 5, n)
r := mongoDoc{}
require.NoError(t, coll.Find(bson.M{"site": "site"}).Sort("+_id").One(&r))
assert.Equal(t, "key-2", r.Key)
return nil
})
}
func TestMongoCache_MaxValueSize(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
lc, err := NewMongoCache(conn, MaxKeys(5), MaxValSize(10))
require.Nil(t, err)
// put good size value to cache and make sure it cached
res, err := lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Z"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res))
res, err = lc.Get(NewKey("site").ID("key-Z"), func() ([]byte, error) {
return []byte("result-Zzzz"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-Z", string(res), "got cached value")
// put too big value to cache and make sure it is not cached
res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) {
return []byte("1234567890"), nil
})
assert.Nil(t, err)
assert.Equal(t, "1234567890", string(res))
res, err = lc.Get(NewKey("site").ID("key-Big"), func() ([]byte, error) {
return []byte("result-big"), nil
})
assert.Nil(t, err)
assert.Equal(t, "result-big", string(res), "got not cached value")
}
func TestMongoCache_Parallel(t *testing.T) {
var coldCalls int32
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
lc, err := NewMongoCache(conn)
require.Nil(t, err)
res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value", string(res))
wg := sync.WaitGroup{}
for i := 0; i < 100; i++ {
wg.Add(1)
i := i
go func() {
defer wg.Done()
r, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
atomic.AddInt32(&coldCalls, 1)
return []byte(fmt.Sprintf("result-%d", i)), nil
})
require.Nil(t, err)
v := string(r)
assert.Equal(t, "value", v, "th=%d", i)
}()
}
wg.Wait()
assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls))
}
func TestMongoCache_Flush(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
lc, err := NewMongoCache(conn)
require.Nil(t, err)
addToCache := func(id string, scopes ...string) {
res, err := lc.Get(NewKey("site").ID(id).Scopes(scopes...), func() ([]byte, error) {
return []byte("value" + id), nil
})
require.Nil(t, err)
require.Equal(t, "value"+id, string(res))
}
init := func() {
lc.Flush(Flusher("site"))
addToCache("key1", "s1", "s2")
addToCache("key2", "s1", "s2", "s3")
addToCache("key3", "s1", "s2", "s3")
addToCache("key4", "s2", "s3")
addToCache("key5", "s2")
addToCache("key6")
addToCache("key7", "s4", "s3")
require.Equal(t, 7, mongoCacheSize(t, conn), "cache init")
}
tbl := []struct {
scopes []string
left int
msg string
}{
{[]string{}, 0, "full flush, no scopes"},
{[]string{"s0"}, 7, "flush wrong scope"},
{[]string{"s1"}, 4, "flush s1 scope"},
{[]string{"s2", "s1"}, 2, "flush s2+s1 scope"},
{[]string{"s1", "s2"}, 2, "flush s1+s2 scope"},
{[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"},
{[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"},
{[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"},
}
for i, tt := range tbl {
init()
lc.Flush(Flusher("site").Scopes(tt.scopes...))
assert.Equal(t, tt.left, mongoCacheSize(t, conn), "keys size, %s #%d", tt.msg, i)
}
}
func TestMongoCache_Scopes(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
assert.NoError(t, err)
defer mongo.RemoveTestCollections(t, conn, "cache")
lc, err := NewMongoCache(conn)
require.Nil(t, err)
res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte("value"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value", string(res))
res, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) {
return []byte("value2"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value2", string(res))
assert.Equal(t, 2, mongoCacheSize(t, conn))
lc.Flush(Flusher("site").Scopes("s1"))
assert.Equal(t, 1, mongoCacheSize(t, conn))
_, err = lc.Get(NewKey("site").ID("key2").Scopes("s2"), func() ([]byte, error) {
assert.Fail(t, "should stay")
return nil, nil
})
assert.Nil(t, err)
res, err = lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte("value-upd"), nil
})
assert.Nil(t, err)
assert.Equal(t, "value-upd", string(res), "was deleted, update")
}
func BenchmarkMongoCache(b *testing.B) {
log.Print("[DEBUG] connect to mongo test instance")
srv, err := mongo.NewServerWithURL(os.Getenv("MONGO_TEST"), 10*time.Second)
assert.Nil(b, err, "failed to dial")
collName := fmt.Sprintf("test_%d", time.Now().Nanosecond())
conn := mongo.NewConnection(srv, "test", collName)
data := ""
for i := 0; i < 1000; i++ {
data += "x"
}
lc, err := NewMongoCache(conn)
require.Nil(b, err)
res, err := lc.Get(NewKey("site").ID("key").Scopes("s1", "s2"), func() ([]byte, error) {
return []byte(data), nil
})
require.Nil(b, err)
require.True(b, strings.HasPrefix(string(res), "xxxx"), string(res))
key := NewKey("site").ID("key").Scopes("s1", "s2")
b.ResetTimer()
for i := 0; i < b.N; i++ {
lc.Get(key, func() ([]byte, error) {
b.Fail()
return nil, nil
})
}
}
func mongoCacheSize(t *testing.T, conn *mongo.Connection) (count int) {
conn.WithCustomCollection("cache", func(coll *mgo.Collection) (e error) {
count, e = coll.Find(bson.M{"site": "site"}).Count()
require.NoError(t, e)
return e
})
return count
}
backend/app/rest/cache/options.go
Vendored
+9 -24
View File
@@ -1,50 +1,35 @@
package cache
import "github.com/pkg/errors"
// Option func type
type Option func(lc *memoryCache) error
type Option func(lc cacheWithOpts) error
// MaxValSize functional option defines the largest value's size allowed to be cached
// By default it is 0, which means unlimited.
func MaxValSize(max int) Option {
return func(lc *memoryCache) error {
lc.maxValueSize = max
if max <= 0 {
return errors.Errorf("negative size for MaxValSize, %d", max)
}
return nil
return func(lc cacheWithOpts) error {
return lc.setMaxValSize(max)
}
}
// MaxKeys functional option defines how many keys to keep.
// By default it is 0, which means unlimited.
func MaxKeys(max int) Option {
return func(lc *memoryCache) error {
lc.maxKeys = max
if max <= 0 {
return errors.Errorf("negative size for MaxKeys, %d", max)
}
return nil
return func(lc cacheWithOpts) error {
return lc.setMaxKeys(max)
}
}
// MaxCacheSize functional option defines the total size of cached data.
// By default it is 0, which means unlimited.
func MaxCacheSize(max int64) Option {
return func(lc *memoryCache) error {
lc.maxCacheSize = max
if max <= 0 {
return errors.Errorf("negative size or MaxCacheSize, %d", max)
}
return nil
return func(lc cacheWithOpts) error {
return lc.setMaxCacheSize(max)
}
}
// PostFlushFn functional option defines how callback function called after each Flush.
func PostFlushFn(postFlushFn func()) Option {
return func(lc *memoryCache) error {
lc.postFlushFn = postFlushFn
return nil
return func(lc cacheWithOpts) error {
return lc.setPostFlushFn(postFlushFn)
}
}
backend/app/rest/proxy/avatar.go
+11 -12
View File
@@ -13,19 +13,18 @@ import (
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
)
// Avatar provides file-system store and http handler for avatars
// Avatar provides http handler for avatars from avatar.Store
// On user login auth will call Put and it will retrieve and save picture locally.
type Avatar struct {
Store AvatarStore
Store avatar.Store
RoutePath string
RemarkURL string
}
const imgSfx = ".image"
// Put stores retrieved avatar to StorePath. Gets image from user info. Returns proxied url
// Put stores retrieved avatar to avatar.Store. Gets image from user info. Returns proxied url
func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
// no picture for user, try default avatar
@@ -55,13 +54,13 @@ func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status)
}
avatar, err := p.Store.Put(u.ID, resp.Body)
avatarID, err := p.Store.Put(u.ID, resp.Body) // put returns avatar base name, like 123456.image
if err != nil {
return "", err
}
log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatar, u.Name)
return p.RemarkURL + p.RoutePath + "/" + avatar, nil
log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatarID, u.Name)
return p.RemarkURL + p.RoutePath + "/" + avatarID, nil
}
// Routes returns auth routes for given provider
@@ -72,10 +71,10 @@ func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string,
// GET /123456789.image
router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) {
avatar := chi.URLParam(r, "avatar")
avatarID := chi.URLParam(r, "avatar")
// enforce client-side caching
etag := `"` + p.Store.ID(avatar) + `"`
etag := `"` + p.Store.ID(avatarID) + `"`
w.Header().Set("Etag", etag)
w.Header().Set("Cache-Control", "max-age=604800") // 7 days
if match := r.Header.Get("If-None-Match"); match != "" {
@@ -85,7 +84,7 @@ func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string,
}
}
avReader, size, err := p.Store.Get(avatar)
avReader, size, err := p.Store.Get(avatarID)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar")
return
@@ -93,7 +92,7 @@ func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string,
defer func() {
if e := avReader.Close(); e != nil {
log.Printf("[WARN] can't close avatar reader for %s, %s", avatar, e)
log.Printf("[WARN] can't close avatar reader for %s, %s", avatarID, e)
}
}()
backend/app/rest/proxy/avatar_store.go
-159
View File
@@ -1,159 +0,0 @@
package proxy
//go:generate sh -c "mockery -inpkg -name AvatarStore -print > /tmp/mock.tmp && mv /tmp/mock.tmp avatar_store_mock.go"
import (
"bytes"
"fmt"
"hash/crc64"
"image"
"image/png"
"io"
"log"
"os"
"path"
"strconv"
"strings"
"sync"
// Initializing packages for supporting GIF and JPEG formats.
_ "image/gif"
_ "image/jpeg"
"github.com/pkg/errors"
"golang.org/x/image/draw"
"github.com/umputun/remark/backend/app/store"
)
// AvatarStore defines interface to store and serve avatars
type AvatarStore interface {
Put(userID string, reader io.Reader) (avatar string, err error)
Get(avatar string) (reader io.ReadCloser, size int, err error)
ID(avatar string) (id string)
}
// FSAvatarStore implements AvatarStore for local file system
type FSAvatarStore struct {
storePath string
resizeLimit int
ctcTable *crc64.Table
once sync.Once
}
// NewFSAvatarStore makes file-system avatar store
func NewFSAvatarStore(storePath string, resizeLimit int) *FSAvatarStore {
return &FSAvatarStore{storePath: storePath, resizeLimit: resizeLimit}
}
// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
func (fs *FSAvatarStore) Put(userID string, reader io.Reader) (avatar string, err error) {
id := store.EncodeID(userID)
location := fs.location(id) // location adds partition to path
if _, err = os.Stat(location); os.IsNotExist(err) {
if e := os.Mkdir(location, 0700); e != nil {
return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
}
}
avFile := path.Join(location, id+imgSfx)
fh, err := os.Create(avFile)
if err != nil {
return "", errors.Wrapf(err, "can't create file %s", avFile)
}
defer func() {
if e := fh.Close(); e != nil {
log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
}
}()
// Trying to resize avatar.
if reader = resize(reader, fs.resizeLimit); reader == nil {
return "", errors.New("avatar reader is nil")
}
if _, err = io.Copy(fh, reader); err != nil {
return "", errors.Wrapf(err, "can't save file %s", avFile)
}
return id + imgSfx, nil
}
// Get avatar reader for avatar id.image
func (fs *FSAvatarStore) Get(avatar string) (reader io.ReadCloser, size int, err error) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fh, err := os.Open(avFile)
if err != nil {
return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar)
}
if fi, e := fh.Stat(); e == nil {
size = int(fi.Size())
}
return fh, size, nil
}
// ID returns a fingerprint of the avatar content.
func (fs *FSAvatarStore) ID(avatar string) (id string) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fi, err := os.Stat(avFile)
if err != nil {
log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err)
return store.EncodeID(avatar)
}
return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10))
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
func (fs *FSAvatarStore) location(id string) string {
fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
partition := checksum64 % 100
return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
}
// Resizes an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
// (width or height) preserving aspect ratio.
// Returns original reader if resizing is not needed or failed.
func resize(reader io.Reader, limit int) io.Reader {
if reader == nil {
log.Print("[WARN] avatar resize(): reader is nil")
return nil
}
if limit <= 0 {
log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
return reader
}
var teeBuf bytes.Buffer
tee := io.TeeReader(reader, &teeBuf)
src, _, err := image.Decode(tee)
if err != nil {
log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
return &teeBuf
}
bounds := src.Bounds()
w, h := bounds.Dx(), bounds.Dy()
if w <= limit && h <= limit || w <= 0 || h <= 0 {
log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
return &teeBuf
}
newW, newH := w*limit/h, limit
if w > h {
newW, newH = limit, h*limit/w
}
m := image.NewRGBA(image.Rect(0, 0, newW, newH))
// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
var out bytes.Buffer
if err = png.Encode(&out, m); err != nil {
log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
return &teeBuf
}
return &out
}
backend/app/rest/proxy/avatar_test.go
+5 -4
View File
@@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/avatar"
)
func TestAvatar_Put(t *testing.T) {
@@ -30,7 +31,7 @@ func TestAvatar_Put(t *testing.T) {
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
@@ -59,13 +60,13 @@ func TestAvatar_PutFailed(t *testing.T) {
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
u := store.User{ID: "user1", Name: "user1 name"}
_, err := p.Put(u)
assert.EqualError(t, err, "no picture for user1")
u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"}
u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:22345/avater/pic"}
_, err = p.Put(u)
require.Error(t, err)
assert.Contains(t, err.Error(), "connect: connection refused")
@@ -89,7 +90,7 @@ func TestAvatar_Routes(t *testing.T) {
}))
defer ts.Close()
p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
backend/app/rest/proxy/image.go
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/rest"
)
@@ -113,7 +114,7 @@ func (p Image) extract(commentHTML string) ([]string, error) {
return result, nil
}
// replace img links in commentHTML with route to proxy with base64 encoded original link
// replace img links in commentHTML with route to proxy, base64 encoded original link
func (p Image) replace(commentHTML string, imgs []string) string {
for _, img := range imgs {
backend/app/rest/user.go
+10
View File
@@ -10,6 +10,16 @@ import (
type contextKey string
// MustGetUserInfo fails if can't extract user data from the request.
// should be called from authed controllers only
func MustGetUserInfo(r *http.Request) store.User {
user, err := GetUserInfo(r)
if err != nil {
panic(err)
}
return user
}
// GetUserInfo returns user from request context
func GetUserInfo(r *http.Request) (user store.User, err error) {
backend/app/rest/user_test.go
+19 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/umputun/remark/backend/app/store"
)
func TestGetUserInfo(t *testing.T) {
func TestUser_GetUserInfo(t *testing.T) {
r, err := http.NewRequest("GET", "http://blah.com", nil)
assert.Nil(t, err)
_, err = GetUserInfo(r)
@@ -19,3 +19,21 @@ func TestGetUserInfo(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
}
func TestUSer_MustGetUserInfo(t *testing.T) {
defer func() {
if r := recover(); r != nil {
t.Log("recovered from panic")
}
}()
r, err := http.NewRequest("GET", "http://blah.com", nil)
assert.Nil(t, err)
_ = MustGetUserInfo(r)
assert.Fail(t, "should panic")
r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
u := MustGetUserInfo(r)
assert.Nil(t, err)
assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
}
backend/app/store/admin/admin.go
+50
View File
@@ -0,0 +1,50 @@
// Package admin defines and implements store for admin-level data like secret key, list of admins and so on
package admin
import (
"errors"
"log"
)
// Store defines interface returning admins info for given site
type Store interface {
Key(siteID string) (key string, err error)
Admins(siteID string) (ids []string)
Email(siteID string) (email string)
}
// StaticStore implements keys.Store with a single, predefined key
type StaticStore struct {
admins []string
email string
key string
}
// Key returns static key for all sites, allows empty site
func (s *StaticStore) Key(siteID string) (key string, err error) {
if s.key == "" {
return "", errors.New("empty key for static key store")
}
return s.key, nil
}
// NewStaticStore makes StaticStore instance with given key
func NewStaticStore(key string, admins []string, email string) *StaticStore {
log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
return &StaticStore{key: key, admins: admins, email: email}
}
// NewStaticKeyStore is a shortcut for making StaticStore for key consumers only
func NewStaticKeyStore(key string) *StaticStore {
return &StaticStore{key: key, admins: []string{}, email: ""}
}
// Admins returns static list of admin's ids, the same for all sites
func (s *StaticStore) Admins(string) (ids []string) {
return s.admins
}
// Email gets static email address
func (s *StaticStore) Email(string) (email string) {
return s.email
}
backend/app/store/admin/admin_test.go
+69
View File
@@ -0,0 +1,69 @@
package admin
import (
"testing"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestStaticStore_Get(t *testing.T) {
var ks Store = NewStaticStore("key123", []string{"123", "xyz"}, "aa@example.com")
k, err := ks.Key("any")
assert.NoError(t, err, "valid store")
assert.Equal(t, "key123", k, "valid site")
a := ks.Admins("any")
assert.Equal(t, []string{"123", "xyz"}, a)
email := ks.Email("blah")
assert.Equal(t, "aa@example.com", email)
ks = NewStaticStore("", []string{"123", "xyz"}, "aa@example.com")
_, err = ks.Key("any")
assert.NotNil(t, err, "invalid (empty key) store")
}
func TestMongoStore_Get(t *testing.T) {
conn, err := mongo.MakeTestConnection(t)
require.NoError(t, err)
var ms Store = NewMongoStore(conn)
recs := []mongoRec{
{"site1", "secret1", []string{"i11", "i12"}, "e1"},
{"site2", "secret2", []string{"i21", "i22"}, "e2"},
}
err = conn.WithCollection(func(coll *mgo.Collection) error {
if e1 := coll.Insert(recs[0]); e1 != nil {
return e1
}
return coll.Insert(recs[1])
})
require.NoError(t, err)
admins := ms.Admins("site1")
assert.Equal(t, []string{"i11", "i12"}, admins)
email := ms.Email("site1")
assert.Equal(t, "e1", email)
key, err := ms.Key("site1")
assert.NoError(t, err)
assert.Equal(t, "secret1", key)
admins = ms.Admins("site2")
assert.Equal(t, []string{"i21", "i22"}, admins)
email = ms.Email("site2")
assert.Equal(t, "e2", email)
key, err = ms.Key("site2")
assert.NoError(t, err)
assert.Equal(t, "secret2", key)
admins = ms.Admins("no-site-in-db")
assert.Equal(t, []string{}, admins)
email = ms.Email("no-site-in-db")
assert.Equal(t, "", email)
_, err = ms.Key("no-site-in-db")
assert.Error(t, err, "can't get secret for site no-site-in-db")
}
backend/app/store/admin/mongo.go
+61
View File
@@ -0,0 +1,61 @@
package admin
import (
"log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
)
// MongoStore implements admin.Store with mongo backend
type MongoStore struct {
connection *mongo.Connection
}
type mongoRec struct {
SiteID string `bson:"site"`
SecretKey string `bson:"secret"`
IDs []string `bson:"admin_ids"`
Email string `bson:"admin_email"`
}
// NewMongoStore makes admin Store for mongo's connection
func NewMongoStore(conn *mongo.Connection) *MongoStore {
log.Printf("[DEBUG] make mongo admin store with %+v", conn)
return &MongoStore{connection: conn}
}
// Key executes find by siteID and returns substructure with secret key
func (m *MongoStore) Key(siteID string) (key string, err error) {
resp := mongoRec{}
err = m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
return resp.SecretKey, errors.Wrapf(err, "can't get secret for site %s", siteID)
}
// Admins executes find by siteID and returns admins ids
func (m *MongoStore) Admins(siteID string) (ids []string) {
resp := mongoRec{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return []string{}
}
return resp.IDs
}
// Email executes find by siteID and returns admin's email
func (m *MongoStore) Email(siteID string) (email string) {
resp := mongoRec{}
err := m.connection.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID}).One(&resp)
})
if err != nil {
return ""
}
return resp.Email
}
backend/app/store/avatar/bolt.go
+144
View File
@@ -0,0 +1,144 @@
package avatar
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"io"
"io/ioutil"
"log"
"github.com/coreos/bbolt"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// BoltDB implements avatar store with bolt
// using separate db (file) with "avatars" bucket to keep image bin and "metas" bucket
// to keep sha1 of picture. avatarID (base file name) used as a key for both.
type BoltDB struct {
fileName string // full path to boltdb
resizeLimit int
db *bolt.DB
}
const avatarsBktName = "avatars"
const metasBktName = "metas"
// NewBoltDB makes bolt avatar store
func NewBoltDB(fileName string, options bolt.Options, resizeLimit int) (*BoltDB, error) {
db, err := bolt.Open(fileName, 0600, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to make boltdb for %s", fileName)
}
err = db.Update(func(tx *bolt.Tx) error {
if _, e := tx.CreateBucketIfNotExists([]byte(avatarsBktName)); e != nil {
return errors.Wrapf(e, "failed to create top level bucket %s", avatarsBktName)
}
_, e := tx.CreateBucketIfNotExists([]byte(metasBktName))
return errors.Wrapf(e, "failed to create top metas bucket %s", metasBktName)
})
if err != nil {
return nil, errors.Wrapf(err, "failed to initialize boltdb db %q buckets", fileName)
}
return &BoltDB{db: db, fileName: fileName, resizeLimit: resizeLimit}, nil
}
// Put avatar to bolt, key by avatarID. Trying to resize image and lso calculates sha1 of the file for ID func
func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
// Trying to resize avatar.
if reader = resize(reader, b.resizeLimit); reader == nil {
return "", errors.New("avatar resize reader is nil")
}
avatarID := id + imgSfx
err = b.db.Update(func(tx *bolt.Tx) error {
buf := &bytes.Buffer{}
if _, err = io.Copy(buf, reader); err != nil {
return errors.Wrapf(err, "can't read avatar %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Put([]byte(avatarID), buf.Bytes()); err != nil {
return errors.Wrapf(err, "can't put to bucket with %s", avatarID)
}
// store sha1 of the image
return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID)))
})
return avatarID, err
}
// Get avatar reader for avatar id.image, avatarID used as the direct key
func (b *BoltDB) Get(avatarID string) (reader io.ReadCloser, size int, err error) {
buf := &bytes.Buffer{}
err = b.db.View(func(tx *bolt.Tx) error {
data := tx.Bucket([]byte(avatarsBktName)).Get([]byte(avatarID))
if data == nil {
return errors.Errorf("can't load avatar %s", avatarID)
}
size, err = buf.Write(data)
return errors.Wrapf(err, "failed to write for %s", avatarID)
})
return ioutil.NopCloser(buf), size, err
}
// ID returns a fingerprint of the avatar content.
func (b *BoltDB) ID(avatarID string) (id string) {
data := []byte{}
err := b.db.View(func(tx *bolt.Tx) error {
if data = tx.Bucket([]byte(metasBktName)).Get([]byte(avatarID)); data == nil {
return errors.Errorf("can't load avatar's id for %s", avatarID)
}
return nil
})
if err != nil { // failed to get ID, use encoded avatarID
log.Printf("[DEBUG] can't get avatar info '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return string(data)
}
// Remove avatar from bolt
func (b *BoltDB) Remove(avatarID string) (err error) {
return b.db.Update(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte(avatarsBktName))
if bkt.Get([]byte(avatarID)) == nil {
return errors.Errorf("avatar key not found, %s", avatarID)
}
if err = tx.Bucket([]byte(avatarsBktName)).Delete([]byte(avatarID)); err != nil {
return errors.Wrapf(err, "can't delete avatar object %s", avatarID)
}
return errors.Wrapf(tx.Bucket([]byte(metasBktName)).Delete([]byte(avatarID)),
"can't delete meta object %s", avatarID)
})
}
// List all avatars (ids) from metas bucket
// note: id includes .image suffix
func (b *BoltDB) List() (ids []string, err error) {
err = b.db.View(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(metasBktName)).ForEach(func(k, _ []byte) error {
ids = append(ids, string(k))
return nil
})
})
return ids, errors.Wrap(err, "failed to list")
}
// Close bolt store
func (b *BoltDB) Close() error {
return errors.Wrapf(b.db.Close(), "failed to close %s", b.fileName)
}
func (b *BoltDB) sha1(data []byte, avatarID string) (id string) {
h := sha1.New()
if _, err := h.Write(data); err != nil {
log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
return store.EncodeID(avatarID)
}
return hex.EncodeToString(h.Sum(nil))
}
backend/app/store/avatar/bolt_test.go
+99
View File
@@ -0,0 +1,99 @@
package avatar
import (
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/coreos/bbolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testDb = "/tmp/test-remark-avatars.db"
func TestBoltDB_PutAndGet(t *testing.T) {
var b Store = prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
rd, size, err := b.Get(avatar)
require.Nil(t, err)
assert.Equal(t, 21, size)
data, err := ioutil.ReadAll(rd)
require.Nil(t, err)
assert.Equal(t, "some picture bin data", string(data))
_, _, err = b.Get("bad avatar")
assert.NotNil(t, err)
// check IDs
assert.Equal(t, "fddae9ce556712a6ece0e8951a6e7a05c51ed6bf", b.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", b.ID("aaaa"), "no data, encoded avatar id")
l, err := b.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestBoltDB_Remove(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
assert.NotNil(t, b.Remove("no-such-thing.image"))
avatar, err := b.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
assert.NoError(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one")
assert.NotNil(t, b.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestBoltDB_List(t *testing.T) {
b := prepBoltStore(t)
defer func() {
assert.Nil(t, b.Close())
os.Remove(testDb)
}()
// write some avatars
_, err := b.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = b.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = b.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := b.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := b.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
// makes new boltdb, put two records
func prepBoltStore(t *testing.T) *BoltDB {
os.Remove(testDb)
boltStore, err := NewBoltDB(testDb, bolt.Options{}, 0)
require.Nil(t, err)
return boltStore
}
backend/app/store/avatar/gridfs.go
+125
View File
@@ -0,0 +1,125 @@
package avatar
import (
"bytes"
"io"
"io/ioutil"
"log"
"time"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// NewGridFS makes gridfs (mongo) avatar store
func NewGridFS(conn *mongo.Connection, resizeLimit int) *GridFS {
return &GridFS{Connection: conn, resizeLimit: resizeLimit}
}
// GridFS implements Store for GridFS
type GridFS struct {
Connection *mongo.Connection
resizeLimit int
}
// Put avatar to gridfs object, try to resize
func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Create(id + imgSfx)
if e != nil {
return e
}
defer func() {
if err = fh.Close(); err != nil {
log.Printf("[WARN] can't close avatar file %v, %s", fh, err)
}
}()
// Trying to resize avatar.
if reader = resize(reader, gf.resizeLimit); reader == nil {
return errors.New("avatar resize reader is nil")
}
_, e = io.Copy(fh, reader)
return e
})
return id + imgSfx, err
}
// Get avatar reader for avatar id.image
func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
buf := &bytes.Buffer{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't load avatar %s", avatar)
}
if _, e = io.Copy(buf, fh); e != nil {
return errors.Wrapf(e, "can't copy avatar %s", avatar)
}
size = int(fh.Size())
return fh.Close()
})
return ioutil.NopCloser(buf), size, err
}
// ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly
func (gf *GridFS) ID(avatar string) (id string) {
err := gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't open avatar %s", avatar)
}
id = fh.MD5()
return errors.Wrapf(fh.Close(), "can't close avatar")
})
if err != nil {
log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err)
return store.EncodeID(avatar)
}
return id
}
// Remove avatar from gridfs
func (gf *GridFS) Remove(avatar string) error {
return gf.Connection.WithDB(func(dbase *mgo.Database) error {
fh, e := dbase.GridFS("fs").Open(avatar)
if e != nil {
return errors.Wrapf(e, "can't get avatar %s", avatar)
}
if e = fh.Close(); e != nil {
log.Printf("[WARN] can't close avatar %s, %s", avatar, e)
}
return dbase.GridFS("fs").Remove(avatar)
})
}
// List all avatars (ids) on gfs
// note: id includes .image suffix
func (gf *GridFS) List() (ids []string, err error) {
type gfsFile struct {
UploadDate time.Time `bson:"uploadDate"`
Length int64 `bson:",minsize"`
MD5 string
Filename string `bson:",omitempty"`
}
files := []gfsFile{}
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
return dbase.GridFS("fs").Find(nil).All(&files)
})
for _, f := range files {
ids = append(ids, f.Filename)
}
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (gf *GridFS) Close() error {
return nil
}
backend/app/store/avatar/gridfs_test.go
+97
View File
@@ -0,0 +1,97 @@
package avatar
import (
"io/ioutil"
"sort"
"strings"
"testing"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGridFS_PutAndGet(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
avatar, err := p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
rd, size, err := p.Get(avatar)
require.Nil(t, err)
assert.Equal(t, 21, size)
data, err := ioutil.ReadAll(rd)
require.Nil(t, err)
assert.Equal(t, "some picture bin data", string(data))
_, _, err = p.Get("bad avatar")
assert.NotNil(t, err)
assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar))
assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id")
l, err := p.List()
require.Nil(t, err)
assert.Equal(t, 1, len(l))
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", l[0])
}
func TestGridFS_Remove(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
assert.NotNil(t, p.Remove("no-such-thing.image"))
avatar, err := p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "remove real one")
assert.NotNil(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"), "already removed")
}
func TestGridFS_List(t *testing.T) {
p, skip := prepGFStore(t)
if skip {
return
}
// write some avatars
_, err := p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func prepGFStore(t *testing.T) (Store, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
_ = conn.WithCustomCollection("fs.chunks", func(coll *mgo.Collection) error {
return coll.DropCollection()
})
_ = conn.WithCustomCollection("fs.files", func(coll *mgo.Collection) error {
return coll.DropCollection()
})
return NewGridFS(conn, 0), false
}
backend/app/store/avatar/localfs.go
+129
View File
@@ -0,0 +1,129 @@
package avatar
import (
"fmt"
"hash/crc64"
"io"
"log"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// LocalFS implements Store for local file system
type LocalFS struct {
storePath string
resizeLimit int
ctcTable *crc64.Table
once sync.Once
}
// NewLocalFS makes file-system avatar store
func NewLocalFS(storePath string, resizeLimit int) *LocalFS {
return &LocalFS{storePath: storePath, resizeLimit: resizeLimit}
}
// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
// userID can be avatarID as well, in this case encoding just strip .image prefix
func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) {
id := encodeID(userID)
location := fs.location(id) // location adds partition to path
if _, err = os.Stat(location); os.IsNotExist(err) {
if e := os.Mkdir(location, 0700); e != nil {
return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
}
}
avFile := path.Join(location, id+imgSfx)
fh, err := os.Create(avFile)
if err != nil {
return "", errors.Wrapf(err, "can't create file %s", avFile)
}
defer func() {
if e := fh.Close(); e != nil {
log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
}
}()
// Trying to resize avatar.
if reader = resize(reader, fs.resizeLimit); reader == nil {
return "", errors.New("avatar resize reader is nil")
}
if _, err = io.Copy(fh, reader); err != nil {
return "", errors.Wrapf(err, "can't save file %s", avFile)
}
return id + imgSfx, nil
}
// Get avatar reader for avatar id.image
func (fs *LocalFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fh, err := os.Open(avFile)
if err != nil {
return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar)
}
if fi, e := fh.Stat(); e == nil {
size = int(fi.Size())
}
return fh, size, nil
}
// ID returns a fingerprint of the avatar content.
func (fs *LocalFS) ID(avatar string) (id string) {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
fi, err := os.Stat(avFile)
if err != nil {
log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err)
return store.EncodeID(avatar)
}
return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10))
}
// Remove avatar file
func (fs *LocalFS) Remove(avatar string) error {
location := fs.location(strings.TrimSuffix(avatar, imgSfx))
avFile := path.Join(location, avatar)
return os.Remove(avFile)
}
// List all avatars (ids) on local file system
// note: id includes .image suffix
func (fs *LocalFS) List() (ids []string, err error) {
err = filepath.Walk(fs.storePath,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), imgSfx) {
ids = append(ids, info.Name())
}
return nil
})
return ids, errors.Wrap(err, "can't list avatars")
}
// Close gridfs does nothing but satisfies interface
func (fs *LocalFS) Close() error {
return nil
}
// get location (directory) for user id by adding partition to final path in order to keep files
// in different subdirectories and avoid too many files in a single place.
// the end result is a full path like this - /tmp/avatars.test/92
func (fs *LocalFS) location(id string) string {
fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
partition := checksum64 % 100
return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
}
backend/app/rest/proxy/avatar_store_test.go → backend/app/store/avatar/localfs_test.go
+68 -68
View File
@@ -1,11 +1,9 @@
package proxy
package avatar
import (
"bytes"
"image"
"io"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"time"
@@ -14,15 +12,15 @@ import (
"github.com/stretchr/testify/require"
)
func TestAvatarStore_Put(t *testing.T) {
p := NewFSAvatarStore("/tmp/avatars.test", 300)
func TestAvatarStoreFS_Put(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
avatar, err := p.Put("user1", nil)
assert.Equal(t, "", avatar)
assert.EqualError(t, err, "avatar reader is nil")
assert.EqualError(t, err, "avatar resize reader is nil")
avatar, err = p.Put("user1", strings.NewReader("some picture bin data"))
require.Nil(t, err)
@@ -38,6 +36,14 @@ func TestAvatarStore_Put(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with encoded id
avatar, err = p.Put("f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", strings.NewReader("some picture bin data 123"))
require.Nil(t, err)
assert.Equal(t, "f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image", avatar)
fi, err = os.Stat("/tmp/avatars.test/56/f1881c06eec96db9901c7bbfe41c42a3f08e9cb8.image")
assert.NoError(t, err)
assert.Equal(t, int64(25), fi.Size())
// with resize
file, e := os.Open("testdata/circles.png")
require.Nil(t, e)
@@ -48,13 +54,13 @@ func TestAvatarStore_Put(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int64(6986), fi.Size())
p = NewFSAvatarStore("/dev/null", 300)
p = NewLocalFS("/dev/null", 300)
_, err = p.Put("user1", strings.NewReader("some picture bin data"))
assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
}
func TestAvatarStore_Get(t *testing.T) {
p := NewFSAvatarStore("/tmp/avatars.test", 300)
func TestAvatarStoreFS_Get(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
@@ -77,8 +83,8 @@ func TestAvatarStore_Get(t *testing.T) {
assert.Equal(t, "something", string(data))
}
func TestAvatarStore_Location(t *testing.T) {
p := NewFSAvatarStore("/tmp/avatars.test", 300)
func TestAvatarStoreFS_Location(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
tbl := []struct {
id string
@@ -87,6 +93,7 @@ func TestAvatarStore_Location(t *testing.T) {
{"abc", "/tmp/avatars.test/35"},
{"xyz", "/tmp/avatars.test/69"},
{"blah blah", "/tmp/avatars.test/29"},
{"f1881c06eec96db9901c7bbfe41c42a3f08e9cb8", "/tmp/avatars.test/56"},
}
for i, tt := range tbl {
@@ -94,60 +101,8 @@ func TestAvatarStore_Location(t *testing.T) {
}
}
func TestAvatarStore_resize(t *testing.T) {
checkC := func(t *testing.T, r io.Reader, cExp []byte) {
content, err := ioutil.ReadAll(r)
require.NoError(t, err)
assert.Equal(t, cExp, content)
}
// Reader is nil.
resizedR := resize(nil, 100)
// assert.EqualError(t, err, "limit should be greater than 0")
assert.Nil(t, resizedR)
// Negative limit error.
resizedR = resize(strings.NewReader("some picture bin data"), -1)
require.NotNil(t, resizedR)
checkC(t, resizedR, []byte("some picture bin data"))
// Decode error.
resizedR = resize(strings.NewReader("invalid image content"), 100)
assert.NotNil(t, resizedR)
checkC(t, resizedR, []byte("invalid image content"))
cases := []struct {
file string
wr, hr int
}{
{"testdata/circles.png", 400, 300}, // full size: 800x600 px
{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
}
for _, c := range cases {
img, err := ioutil.ReadFile(c.file)
require.Nil(t, err, "can't open test file %s", c.file)
// No need for resize, avatar dimensions are smaller than resize limit.
resizedR = resize(bytes.NewReader(img), 800)
assert.NotNilf(t, resizedR, "file %s", c.file)
checkC(t, resizedR, img)
// Resizing to half of width. Check resizedR avatar format PNG.
resizedR = resize(bytes.NewReader(img), 400)
assert.NotNilf(t, resizedR, "file %s", c.file)
imgRz, format, err := image.Decode(resizedR)
assert.Nilf(t, err, "file %s", c.file)
assert.Equalf(t, "png", format, "file %s", c.file)
bounds := imgRz.Bounds()
assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
}
}
func TestAvatarStore_ID(t *testing.T) {
p := NewFSAvatarStore("/tmp/avatars.test", 300)
func TestAvatarStoreFS_ID(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
@@ -164,8 +119,53 @@ func TestAvatarStore_ID(t *testing.T) {
id = p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.Equal(t, "325d5b451f32c2f8e7f30a9fd65bff6a42954d9a", id) // store.EncodeID("b3daa77b4c04a9551b8781d03191fe098f325e67.image1500000000")
}
func BenchmarkAvatarStore_ID(b *testing.B) {
p := NewFSAvatarStore("/tmp/avatars.test", 300)
func TestAvatarStoreFS_Remove(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
assert.NotNil(t, p.Remove("no-such-avatar"), "remove non-existing avatar")
err = ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
require.NoError(t, err)
assert.NoError(t, p.Remove("b3daa77b4c04a9551b8781d03191fe098f325e67.image"))
_, err = os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.NotNil(t, err, "removed for real")
t.Log(err)
}
func TestAvatarStoreFS_List(t *testing.T) {
p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// write some avatars
_, err = p.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = p.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = p.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
l, err := p.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
r, size, err := p.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
func BenchmarkAvatarStoreFS_ID(b *testing.B) {
p := NewLocalFS("/tmp/avatars.test", 300)
os.MkdirAll("/tmp/avatars.test/30", 0700)
defer os.RemoveAll("/tmp/avatars.test")
err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
backend/app/store/avatar/store.go
+110
View File
@@ -0,0 +1,110 @@
// Package avatar defines store interface and implements local (fs), gridfs (mongo) and boltdb stores.
//
package avatar
//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go"
import (
"bytes"
"image"
"strings"
// Initializing packages for supporting GIF and JPEG formats.
_ "image/gif"
_ "image/jpeg"
"image/png"
"io"
"log"
"regexp"
"github.com/umputun/remark/backend/app/store"
"golang.org/x/image/draw"
)
// imgSfx for avatars
const imgSfx = ".image"
var reValidAvatarID = regexp.MustCompile(`^[a-fA-F0-9]{40}\.image$`)
// Store defines interface to store and and load avatars
type Store interface {
Put(userID string, reader io.Reader) (avatarID string, err error) // save avatar data from the reader and return base name
Get(avatarID string) (reader io.ReadCloser, size int, err error) // load avatar via reader
ID(avatarID string) (id string) // unique id of stored avatar's data
Remove(avatarID string) error // remove avatar data
List() (ids []string, err error) // list all avatar ids
Close() error // close store
}
// Migrate avatars between stores
func Migrate(dst Store, src Store) (int, error) {
ids, err := src.List()
if err != nil {
return 0, err
}
for _, id := range ids {
srcReader, _, err := src.Get(id)
if err != nil {
log.Printf("[WARN] can't get reader for avatar %s", id)
continue
}
if _, err = dst.Put(id, srcReader); err != nil {
log.Printf("[WARN] can't put avatar %s", id)
}
if err = srcReader.Close(); err != nil {
log.Printf("[WARN] failed to close avatar %s", id)
}
}
return len(ids), nil
}
// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
// (width or height) preserving aspect ratio.
// Returns original reader if resizing is not needed or failed.
func resize(reader io.Reader, limit int) io.Reader {
if reader == nil {
log.Print("[WARN] avatar resize(): reader is nil")
return nil
}
if limit <= 0 {
log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
return reader
}
var teeBuf bytes.Buffer
tee := io.TeeReader(reader, &teeBuf)
src, _, err := image.Decode(tee)
if err != nil {
log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
return &teeBuf
}
bounds := src.Bounds()
w, h := bounds.Dx(), bounds.Dy()
if w <= limit && h <= limit || w <= 0 || h <= 0 {
log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
return &teeBuf
}
newW, newH := w*limit/h, limit
if w > h {
newW, newH = limit, h*limit/w
}
m := image.NewRGBA(image.Rect(0, 0, newW, newH))
// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
var out bytes.Buffer
if err = png.Encode(&out, m); err != nil {
log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
return &teeBuf
}
return &out
}
// encodeID converts string to encoded id unless already encoded and valid avatar id (with .image) passed
func encodeID(val string) string {
if reValidAvatarID.MatchString(val) {
return strings.TrimSuffix(val, imgSfx) // already encoded, strip .image
}
return store.EncodeID(val)
}
backend/app/store/avatar/store_test.go
+109
View File
@@ -0,0 +1,109 @@
package avatar
import (
"bytes"
"image"
"io"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAvatarStore_resize(t *testing.T) {
checkC := func(t *testing.T, r io.Reader, cExp []byte) {
content, err := ioutil.ReadAll(r)
require.NoError(t, err)
assert.Equal(t, cExp, content)
}
// Reader is nil.
resizedR := resize(nil, 100)
// assert.EqualError(t, err, "limit should be greater than 0")
assert.Nil(t, resizedR)
// Negative limit error.
resizedR = resize(strings.NewReader("some picture bin data"), -1)
require.NotNil(t, resizedR)
checkC(t, resizedR, []byte("some picture bin data"))
// Decode error.
resizedR = resize(strings.NewReader("invalid image content"), 100)
assert.NotNil(t, resizedR)
checkC(t, resizedR, []byte("invalid image content"))
cases := []struct {
file string
wr, hr int
}{
{"testdata/circles.png", 400, 300}, // full size: 800x600 px
{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
}
for _, c := range cases {
img, err := ioutil.ReadFile(c.file)
require.Nil(t, err, "can't open test file %s", c.file)
// No need for resize, avatar dimensions are smaller than resize limit.
resizedR = resize(bytes.NewReader(img), 800)
assert.NotNilf(t, resizedR, "file %s", c.file)
checkC(t, resizedR, img)
// Resizing to half of width. Check resizedR avatar format PNG.
resizedR = resize(bytes.NewReader(img), 400)
assert.NotNilf(t, resizedR, "file %s", c.file)
imgRz, format, err := image.Decode(resizedR)
assert.Nilf(t, err, "file %s", c.file)
assert.Equalf(t, "png", format, "file %s", c.file)
bounds := imgRz.Bounds()
assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
}
}
func TestAvatarStore_Migrate(t *testing.T) {
// prep localfs
plocal := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
// prep gridfs
pgfs, skip := prepGFStore(t)
if skip {
return
}
// write to localfs
_, err = plocal.Put("user1", strings.NewReader("some picture bin data 1"))
require.Nil(t, err)
_, err = plocal.Put("user2", strings.NewReader("some picture bin data 2"))
require.Nil(t, err)
_, err = plocal.Put("user3", strings.NewReader("some picture bin data 3"))
require.Nil(t, err)
// migrate and check reported count
count, err := Migrate(pgfs, plocal)
require.NoError(t, err)
assert.Equal(t, 3, count, "all 3 recs migrated")
// list avatars
l, err := pgfs.List()
assert.NoError(t, err)
assert.Equal(t, 3, len(l), "3 avatars listed in destination store")
sort.Strings(l)
assert.Equal(t, []string{"0b7f849446d3383546d15a480966084442cd2193.image", "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", "b3daa77b4c04a9551b8781d03191fe098f325e67.image"}, l)
// try to read one of migrated avatars
r, size, err := pgfs.Get("0b7f849446d3383546d15a480966084442cd2193.image")
assert.Nil(t, err)
assert.Equal(t, 23, size)
data, err := ioutil.ReadAll(r)
assert.Nil(t, err)
assert.Equal(t, "some picture bin data 3", string(data))
}
backend/app/rest/proxy/testdata/circles.jpg → backend/app/store/avatar/testdata/circles.jpg
Vendored
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

backend/app/rest/proxy/testdata/circles.png → backend/app/store/avatar/testdata/circles.png
Vendored
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

backend/app/store/comment.go
+11 -49
View File
@@ -2,18 +2,15 @@ package store
import (
"html/template"
"net/url"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/microcosm-cc/bluemonday"
)
// Comment represents a single comment with optional reference to its parent
type Comment struct {
ID string `json:"id"`
ID string `json:"id" bson:"_id"`
ParentID string `json:"pid"`
Text string `json:"text"`
Orig string `json:"orig,omitempty"`
@@ -21,21 +18,21 @@ type Comment struct {
Locator Locator `json:"locator"`
Score int `json:"score"`
Votes map[string]bool `json:"votes"`
Timestamp time.Time `json:"time"`
Edit *Edit `json:"edit,omitempty"` // pointer to have empty default in json response
Pin bool `json:"pin,omitempty"`
Deleted bool `json:"delete,omitempty"`
Timestamp time.Time `json:"time" bson:"time"`
Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
Pin bool `json:"pin,omitempty" bson:"pin,omitempty"`
Deleted bool `json:"delete,omitempty" bson:"delete"`
}
// Locator keeps site and url of the post
type Locator struct {
SiteID string `json:"site,omitempty"`
SiteID string `json:"site,omitempty" bson:"site"`
URL string `json:"url"`
}
// Edit indication
type Edit struct {
Timestamp time.Time `json:"time"`
Timestamp time.Time `json:"time" bson:"time"`
Summary string `json:"summary"`
}
@@ -43,9 +40,9 @@ type Edit struct {
type PostInfo struct {
URL string `json:"url"`
Count int `json:"count"`
ReadOnly bool `json:"read_only,omitempty"`
FirstTS time.Time `json:"first_time,omitempty"`
LastTS time.Time `json:"last_time,omitempty"`
ReadOnly bool `json:"read_only,omitempty" bson:"read_only,omitempty"`
FirstTS time.Time `json:"first_time,omitempty" bson:"first_time,omitempty"`
LastTS time.Time `json:"last_time,omitempty" bson:"last_time,omitempty"`
}
// BlockedUser holds id and ts for blocked user
@@ -97,48 +94,13 @@ func (c *Comment) SetDeleted(mode DeleteMode) {
}
}
// Sanitize clean dangerous html/js from the comment, shorten autolinks.
// Sanitize clean dangerous html/js from the comment
func (c *Comment) Sanitize() {
p := bluemonday.UGCPolicy()
p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
c.Text = p.Sanitize(c.Text)
c.Text = shortenAutoLinks(c.Text, shortURLLen)
c.Orig = p.Sanitize(c.Orig)
c.User.ID = template.HTMLEscapeString(c.User.ID)
c.User.Name = template.HTMLEscapeString(c.User.Name)
c.User.Picture = p.Sanitize(c.User.Picture)
}
// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
func shortenAutoLinks(commentHTML string, max int) (resHTML string) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
if err != nil {
return commentHTML
}
doc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok {
if href != s.Text() || len(href) < max+3 || max < 3 {
return
}
url, e := url.Parse(href)
if e != nil {
return
}
url.Path, url.RawQuery, url.Fragment = "", "", ""
host := url.String()
if host == "" {
return
}
short := href[:max-3]
if len(short) < len(host) {
short = host
}
s.SetText(short + "...")
}
})
resHTML, err = doc.Find("body").Html()
if err != nil {
return commentHTML
}
return resHTML
}
backend/app/store/comment_test.go
+6 -49
View File
@@ -17,21 +17,21 @@ func TestComment_Sanitize(t *testing.T) {
{
inp: Comment{
Text: `blah <a href="javascript:alert('XSS1')" onmouseover="alert('XSS2')">XSS</a>` + "\n\t",
User: User{ID: `<a href="http://blah.com">username</a>`},
User: User{ID: `<a href="http://blah.com">username</a>`, Name: "name <b/>"},
},
out: Comment{
Text: "blah XSS\n\t",
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`, Name: "name &lt;b/&gt;"},
},
},
{
inp: Comment{
Text: `blah <a href="https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/">https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/</a>` + "\n\t",
User: User{ID: `<a href="http://blah.com">username</a>`},
Text: "blah 123" + "\n\t",
User: User{ID: "id", Name: "xyz"},
},
out: Comment{
Text: `blah <a href="https://www.reddit.com/r/golang/comments/8jdo2l/remark42_is_a_selfhosted_lightweight_and_simple/" rel="nofollow">https://www.reddit.com/r/golang/comments/8jdo...</a>` + "\n\t",
User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
Text: `blah 123` + "\n\t",
User: User{ID: "id", Name: "xyz"},
},
},
}
@@ -120,46 +120,3 @@ func TestComment_SetDeletedHard(t *testing.T) {
assert.False(t, comment.Pin)
assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
}
func TestComment_ShortenAutoLinks(t *testing.T) {
tbl := []struct {
max int
in, out string
}{
{32, "", ""},
{32, "text", "text"},
{32, "<p>asd</p>", "<p>asd</p>"},
{5, `<a href="incorrect-url">incorrect-url</a>`, `<a href="incorrect-url">incorrect-url</a>`},
{32, `<a href="https://blah.com">some text, not href</a>`, `<a href="https://blah.com">some text, not href</a>`},
{
32,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
{
31,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=1...</a>`,
},
{
15,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
3,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
-1,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
}
for n, tt := range tbl {
got := shortenAutoLinks(tt.in, tt.max)
assert.Equalf(t, tt.out, got, "check #%d", n)
}
}
backend/app/store/engine/bolt_accessor.go
+12 -5
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/coreos/bbolt"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
@@ -37,13 +38,9 @@ const (
readonlyBucketName = "readonly"
verifiedBucketName = "verified"
// limits
lastLimit = 1000
userLimit = 500
tsNano = "2006-01-02T15:04:05.000000000Z07:00"
)
const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
// BoltSite defines single site param
type BoltSite struct {
FileName string // full path to boltdb
@@ -411,6 +408,16 @@ func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
})
}
// Close boltdb store
func (b *BoltDB) Close() error {
errs := new(multierror.Error)
for site, db := range b.dbs {
err := errors.Wrapf(db.Close(), "can't close site %s", site)
errs = multierror.Append(errs, err)
}
return errs.ErrorOrNil()
}
// getPostBucket return bucket with all comments for postURL
func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
postsBkt := tx.Bucket([]byte(postsBucketName))
backend/app/store/engine/bolt_accessor_test.go
+2
View File
@@ -32,6 +32,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
assert.EqualError(t, err, `site "radio-t-bad" not found`)
assert.NoError(t, b.Close())
}
func TestBoltDB_CreateReadOnly(t *testing.T) {
backend/app/store/engine/bolt_admin.go
+1 -1
View File
@@ -266,7 +266,7 @@ func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
})
}
// IsReadOnly checks if user blocked
// IsReadOnly checks if post in RO mode
func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
bdb, err := b.db(locator.SiteID)
backend/app/store/engine/engine.go
+7
View File
@@ -38,6 +38,7 @@ type Accessor interface {
Count(locator store.Locator) (int, error) // number of comments for the post
List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
Close() error // close/stop engine
}
// Admin defines all store ops avail for admin only
@@ -54,6 +55,12 @@ type Admin interface {
IsVerified(siteID string, userID string) bool // check verified status
}
const (
// limits
lastLimit = 1000
userLimit = 500
)
// sortComments is for engines can't sort data internally
func sortComments(comments []store.Comment, sortFld string) []store.Comment {
sort.Slice(comments, func(i, j int) bool {
backend/app/store/engine/mongo.go
+362
View File
@@ -0,0 +1,362 @@
package engine
import (
"time"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
)
// Mongo implements engine interface
type Mongo struct {
conn *mongo.Connection
postWriter mongo.BufferedWriter
}
const (
mongoPosts = "posts"
mongoMetaPosts = "meta_posts"
mongoMetaUsers = "meta_users"
)
type metaPost struct {
ID string `bson:"_id"` // url
SiteID string `bson:"site"`
ReadOnly bool `bson:"read_only"`
}
type metaUser struct {
ID string `bson:"_id"` // user_id
SiteID string `bson:"site"`
Verified bool `bson:"verified"`
Blocked bool `bson:"blocked"`
BlockedUntil time.Time `bson:"blocked_until"`
}
// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
result := Mongo{conn: conn, postWriter: writer}
err := result.prepare()
return &result, errors.Wrap(err, "failed to prepare mongo")
}
// Create new comment, write can be buffered and delayed.
func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
// err = m.postWriter.Write(comment)
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
return coll.Insert(&comment)
})
return comment.ID, err
}
// Find returns all comments for post and sorts results
func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
comments = []store.Comment{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
return coll.Find(query).Sort(sortFld).All(&comments)
})
return comments, err
}
// Get returns comment for locator.URL and commentID string
func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
return coll.Find(query).One(&comment)
})
return comment, err
}
// Put updates comment for locator.URL with mutable part of comment
func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
bson.M{"$set": bson.M{
"text": comment.Text,
"orig": comment.Orig,
"score": comment.Score,
"votes": comment.Votes,
"pin": comment.Pin,
"deleted": comment.Deleted,
}})
})
}
// Last returns up to max last comments for given siteID
func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
comments = []store.Comment{}
if max > lastLimit || max == 0 {
max = lastLimit
}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": siteID, "delete": false}
return coll.Find(query).Sort("-time").Limit(max).All(&comments)
})
return comments, err
}
// Count returns number of comments for locator
func (m *Mongo) Count(locator store.Locator) (count int, err error) {
e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
count, err = coll.Find(query).Count()
return err
})
return count, e
}
// List returns list of all commented posts with counters
func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
list = []store.PostInfo{}
if limit <= 0 {
limit = 1000
}
if skip < 0 {
skip = 0
}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
pipeline := coll.Pipe([]bson.M{
{"$match": bson.M{"locator.site": siteID}},
{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
{"$skip": skip},
{"$limit": limit},
})
return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
})
return list, errors.Wrap(err, "can't get list")
}
// Info returns time range and count for locator
func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
list := []store.PostInfo{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
pipeline := coll.Pipe([]bson.M{
{"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
})
return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
})
if err != nil {
return info, err
}
if len(list) == 0 {
return info, errors.Errorf("can't load info for %s", locator.URL)
}
info = list[0]
// set read-only from age and manual bucket
info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
if m.IsReadOnly(locator) {
info.ReadOnly = true
}
return info, nil
}
// User extracts all comments for given site and given userID
func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
comments = []store.Comment{}
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
query := bson.M{"locator.site": siteID, "user.id": userID}
return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
})
return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
}
// UserCount returns number of comments for user
func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
var e error
count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
return e
})
return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
}
// SetReadOnly makes post read-only or reset the ro flag
func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
return e
})
}
// IsReadOnly checks if post in RO
func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
meta := metaPost{}
err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
})
return err == nil && meta.ReadOnly
}
// SetVerified makes user verified or reset the flag
func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
return e
})
}
// IsVerified checks if user verified
func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
meta := metaUser{}
err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
})
return err == nil && meta.Verified
}
// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
// block uses blocksBucketName with key=userID and val=TTL+now
func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
until := time.Time{}
if status {
until = time.Now().AddDate(100, 0, 0) // permanent is 50year
if ttl > 0 {
until = time.Now().Add(ttl)
}
}
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
return errors.Wrapf(e, "failed to set block for %s", userID)
})
}
// IsBlocked checks if user blocked
func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
meta := metaUser{}
err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
})
return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
}
// Blocked get lists of blocked users for given site
func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
users = []store.BlockedUser{}
metas := []metaUser{}
err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
return coll.Find(bson.M{"site": siteID,
"blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
})
if err != nil {
return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
}
for _, mu := range metas {
blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
blockedUser.Name = ucc[0].User.Name
}
users = append(users, blockedUser)
}
return users, nil
}
// Delete removes comment, by locator from the store.
// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
comment := store.Comment{}
err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
if e != nil {
return e
}
comment.SetDeleted(mode)
return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
})
return errors.Wrapf(err, "can't delete %s", commentID)
}
// DeleteAll removes all info about siteID
func (m *Mongo) DeleteAll(siteID string) error {
err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
_, e := coll.RemoveAll(bson.M{"locator.site": siteID})
return e
})
return errors.Wrapf(err, "can't delete site %s", siteID)
}
// DeleteUser removes all comments for given user. Everything will be market as deleted
// and user name and userID will be changed to "deleted".
func (m *Mongo) DeleteUser(siteID string, userID string) error {
comments := []store.Comment{}
return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
if e != nil {
return e
}
for _, c := range comments {
if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
return e
}
}
return nil
})
}
// Close boltdb store
func (m *Mongo) Close() error {
if m.postWriter != nil {
return m.postWriter.Close()
}
return nil
}
// prepare collections with all indexes
func (m *Mongo) prepare() error {
errs := new(multierror.Error)
e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
})
if e != nil {
return e
}
e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
})
if e != nil {
return e
}
return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
})
}
func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
if limit <= 0 {
limit = 1000
}
if skip < 0 {
skip = 0
}
return q.Skip(skip).Limit(limit)
}
backend/app/store/engine/mongo_test.go
+582
View File
@@ -0,0 +1,582 @@
package engine
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
)
func TestMongo_CreateAndFind(t *testing.T) {
var m Interface
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
require.Equal(t, 2, len(res))
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
assert.Equal(t, "user1", res[0].User.ID)
t.Log(res[0].ID)
_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
assert.NotNil(t, err, "reject dup")
id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
assert.Nil(t, err)
assert.Equal(t, "id-3", id)
res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
assert.Nil(t, err)
require.Equal(t, 1, len(res))
assert.NoError(t, m.Close())
}
func TestMongo_Get(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
assert.Nil(t, err)
assert.Equal(t, "some text2", comment.Text)
comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
assert.NotNil(t, err, "not found")
}
func TestMongo_Put(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
comment := res[0]
comment.Text = "abc 123"
comment.Score = 100
err = m.Put(loc, comment)
assert.Nil(t, err)
comment, err = m.Get(loc, res[0].ID)
assert.Nil(t, err)
assert.Equal(t, "abc 123", comment.Text)
assert.Equal(t, res[0].ID, comment.ID)
assert.Equal(t, 100, comment.Score)
err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
assert.EqualError(t, err, `not found`)
err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
assert.EqualError(t, err, `not found`)
}
func TestMongo_Last(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.Last("radio-t", 0)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text)
res, err = m.Last("radio-t", 1)
assert.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "some text2", res[0].Text)
}
func TestMongo_Count(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 2, c)
c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c)
}
func TestMongo_List(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := m.Create(comment)
assert.Nil(t, err)
ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
res, err := m.List("radio-t", 0, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
res)
res, err = m.List("radio-t", -1, -1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
res, err = m.List("radio-t", 1, 0)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
res, err = m.List("radio-t", 1, 1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
res, err = m.List("bad", 1, 1)
assert.Nil(t, err)
assert.Equal(t, []store.PostInfo{}, res)
}
func TestMongo_Info(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
// add one more for https://radio-t.com/2
comment := store.Comment{
ID: "12345",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
_, err := m.Create(comment)
assert.Nil(t, err)
r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
_, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
require.NotNil(t, err)
_, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
require.NotNil(t, err)
err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
require.Nil(t, err)
r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
require.Nil(t, err)
assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
}
func TestMongo_ReadOnly(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
}
func TestMongo_Verified(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
assert.NoError(t, m.SetVerified("radio-t", "u1", true))
assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
assert.NoError(t, m.SetVerified("radio-t", "u1", false))
assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
}
func TestMongo_GetForUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
res, err := m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = m.User("radio-t", "user1", 1, 0)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
res, err = m.User("radio-t", "user1", 1, 1)
assert.Nil(t, err)
assert.Equal(t, 1, len(res), "allow 1 comment")
assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
res, err = m.User("bad", "user1", 1, 0)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestMongo_GetForUserPagination(t *testing.T) {
m, skip := prepMongo(t, false)
if skip {
return
}
c := store.Comment{
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
// write 200 comments
for i := 0; i < 200; i++ {
c.ID = fmt.Sprintf("id-%d", i)
c.Text = fmt.Sprintf("text #%d", i)
c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
_, err := m.Create(c)
require.Nil(t, err, c.ID)
}
// get all comments
res, err := m.User("radio-t", "user1", 0, 0)
assert.Nil(t, err)
assert.Equal(t, 200, len(res))
assert.Equal(t, "id-199", res[0].ID)
// seek 0, 5 comments
res, err = m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-199", res[0].ID)
assert.Equal(t, "id-195", res[4].ID)
// seek 10, 3 comments
res, err = m.User("radio-t", "user1", 3, 10)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, "id-189", res[0].ID)
assert.Equal(t, "id-187", res[2].ID)
// seek 195, ask 10 comments
res, err = m.User("radio-t", "user1", 10, 195)
assert.Nil(t, err)
assert.Equal(t, 5, len(res))
assert.Equal(t, "id-4", res[0].ID)
assert.Equal(t, "id-0", res[4].ID)
// seek 255, ask 10 comments
res, err = m.User("radio-t", "user1", 10, 255)
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func TestMongo_BlockUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
}
func TestMongo_BlockUserWithTTL(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
time.Sleep(500 * time.Millisecond)
assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
}
func TestMongo_GetForUserCounter(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
count, err := m.UserCount("radio-t", "user1")
assert.Nil(t, err)
assert.Equal(t, 2, count)
count, err = m.UserCount("bad", "user1")
assert.Nil(t, err)
assert.Equal(t, 0, count)
}
func TestMongo_BlockList(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
ids, err := m.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 2, len(ids))
assert.Equal(t, "user1", ids[0].ID)
assert.Equal(t, "user2", ids[1].ID)
t.Logf("%+v", ids)
time.Sleep(500 * time.Millisecond)
ids, err = m.Blocked("radio-t")
assert.NoError(t, err)
assert.Equal(t, 1, len(ids))
assert.Equal(t, "user1", ids[0].ID)
ids, err = m.Blocked("bad")
assert.NoError(t, err)
assert.Equal(t, 0, len(ids))
}
func TestMongo_Delete(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.Nil(t, err)
res, err = m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, "some text2", res[1].Text)
assert.False(t, res[1].Deleted)
comments, err := m.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
err = m.Delete(loc, "123456", store.SoftDelete)
assert.NotNil(t, err)
loc.SiteID = "bad"
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `can't delete id-1: not found`)
loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
err = m.Delete(loc, res[0].ID, store.SoftDelete)
assert.EqualError(t, err, `can't delete id-1: not found`)
}
func TestMongo_DeleteHard(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.Delete(loc, res[0].ID, store.HardDelete)
assert.Nil(t, err)
res, err = m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, "", res[0].Text)
assert.True(t, res[0].Deleted, "marked deleted")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
}
func TestMongo_DeleteAll(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "initially 2 comments")
err = m.DeleteAll("radio-t")
assert.Nil(t, err)
comments, err := m.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
}
func TestMongo_DeleteUser(t *testing.T) {
m, skip := prepMongo(t, true) // adds two comments
if skip {
return
}
err := m.DeleteUser("radio-t", "user1")
require.NoError(t, err)
loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
res, err := m.Find(loc, "time")
assert.Nil(t, err)
assert.Equal(t, 2, len(res), "2 comments with deleted info")
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
assert.Nil(t, err)
assert.Equal(t, 0, c, "0 count")
cc, err := m.User("radio-t", "user1", 5, 0)
assert.Nil(t, err, "no comments for user user1 in store")
assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
comments, err := m.Last("radio-t", 10)
assert.Nil(t, err)
assert.Equal(t, 0, len(comments), "nothing left")
}
func TestMongo_Parallel(t *testing.T) {
var m Interface
var skip bool
m, skip = prepMongoBuffered(t) // buffered engine, no comments
if skip {
return
}
go func() {
for i := 0; i < 100; i++ {
_, err := m.Create(store.Comment{
ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
require.Nil(t, err)
time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
}
}()
for {
time.Sleep(10 * time.Millisecond)
res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
assert.Nil(t, err)
if len(res) == 100 {
break
}
}
}
func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
mongo.RemoveTestCollection(t, conn)
m, err := NewMongo(conn, 1, 0*time.Microsecond)
require.Nil(t, err)
mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
comment := store.Comment{
ID: "id-1",
Text: `some text, <a href="http://radio-t.com">link</a>`,
Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
if writeRecs {
_, err = m.Create(comment)
assert.Nil(t, err)
}
comment = store.Comment{
ID: "id-2",
Text: "some text2",
Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
if writeRecs {
_, err = m.Create(comment)
assert.Nil(t, err)
}
return m, false
}
func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
conn, err := mongo.MakeTestConnection(t)
if err != nil {
return nil, true
}
mongo.RemoveTestCollection(t, conn)
m, err := NewMongo(conn, 10, 10*time.Millisecond)
mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
require.Nil(t, err)
return m, false
}
backend/app/store/formatter.go
+87
View File
@@ -0,0 +1,87 @@
package store
import (
"net/url"
"strings"
"github.com/PuerkitoBio/goquery"
blackfriday "gopkg.in/russross/blackfriday.v2"
)
// CommentFormatter implements all generic formatting ops on comment
type CommentFormatter struct {
converters []CommentConverter
}
// CommentConverter defines interface to convert some parts of commentHTML
// Passed at creation time and does client-defined conversions, like image proxy link change
type CommentConverter interface {
Convert(text string) string
}
// CommentConverterFunc functional struct implementing CommentConverter
type CommentConverterFunc func(text string) string
// Convert calls func for given text
func (f CommentConverterFunc) Convert(text string) string {
return f(text)
}
// NewCommentFormatter makes CommentFormatter
func NewCommentFormatter(converters ...CommentConverter) *CommentFormatter {
return &CommentFormatter{converters: converters}
}
// Format comment fields
func (f *CommentFormatter) Format(c Comment) Comment {
c.Text = f.FormatText(c.Text)
return c
}
// FormatText converts text with markdown processor, applies external converters and shortens links
func (f *CommentFormatter) FormatText(txt string) (res string) {
mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
blackfriday.BackslashLineBreak | blackfriday.Autolink
res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt)))
for _, conv := range f.converters {
res = conv.Convert(res)
}
res = f.shortenAutoLinks(res, shortURLLen)
return res
}
// Shortens all the automatic links in HTML: auto link has equal "href" and "text" attributes.
func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTML string) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
if err != nil {
return commentHTML
}
doc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok {
if href != s.Text() || len(href) < max+3 || max < 3 {
return
}
commentURL, e := url.Parse(href)
if e != nil {
return
}
commentURL.Path, commentURL.RawQuery, commentURL.Fragment = "", "", ""
host := commentURL.String()
if host == "" {
return
}
short := href[:max-3]
if len(short) < len(host) {
short = host
}
s.SetText(short + "...")
}
})
resHTML, err = doc.Find("body").Html()
if err != nil {
return commentHTML
}
return resHTML
}
backend/app/store/formatter_test.go
+104
View File
@@ -0,0 +1,104 @@
package store
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
type mockConverter struct{}
func (m mockConverter) Convert(text string) string { return text + "!converted" }
func TestFormatter_FormatText(t *testing.T) {
tbl := []struct {
in, out string
}{
{"", "!converted"},
{"12345 abc", "<p>12345 abc</p>\n!converted"},
{"**xyz** _aaa_", "<p><strong>xyz</strong> <em>aaa</em></p>\n!converted"},
{
"http://127.0.0.1/some-long-link/12345/678901234567890", "<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0.1/some-long-link/12345/6789012...</a></p>\n!converted",
},
}
f := NewCommentFormatter(mockConverter{})
for n, tt := range tbl {
assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n)
}
}
func TestFormatter_FormatTextNoConverter(t *testing.T) {
f := NewCommentFormatter()
assert.Equal(t, "<p>12345</p>\n", f.FormatText("12345"))
}
func TestFormatter_FormatTextConverterFunc(t *testing.T) {
fn := CommentConverterFunc(func(text string) string { return "zz!" + text })
f := NewCommentFormatter(fn)
assert.Equal(t, "zz!<p>12345</p>\n", f.FormatText("12345"))
}
func TestFormatter_FormatComment(t *testing.T) {
comment := Comment{
Text: "blah\n\nxyz",
User: User{ID: "username"},
ParentID: "p123",
ID: "123",
Locator: Locator{SiteID: "site", URL: "url"},
Score: 10,
Pin: true,
Deleted: true,
Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
Votes: map[string]bool{"uu": true},
}
f := NewCommentFormatter(mockConverter{})
exp := comment
exp.Text = "<p>blah</p>\n\n<p>xyz</p>\n!converted"
assert.Equal(t, exp, f.Format(comment))
}
func TestFormatter_ShortenAutoLinks(t *testing.T) {
f := NewCommentFormatter(nil)
tbl := []struct {
max int
in, out string
}{
{32, "", ""},
{32, "text", "text"},
{32, "<p>asd</p>", "<p>asd</p>"},
{5, `<a href="incorrect-url">incorrect-url</a>`, `<a href="incorrect-url">incorrect-url</a>`},
{32, `<a href="https://blah.com">some text, not href</a>`, `<a href="https://blah.com">some text, not href</a>`},
{
32,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
{
31,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=1...</a>`,
},
{
15,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
3,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com...</a>`,
},
{
-1,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
`<a href="https://blah.com/a/b/c/d?g=123#anc">https://blah.com/a/b/c/d?g=123#anc</a>`,
},
}
for n, tt := range tbl {
got := f.shortenAutoLinks(tt.in, tt.max)
assert.Equalf(t, tt.out, got, "check #%d", n)
}
}
backend/app/store/service/service.go
+28 -9
View File
@@ -8,6 +8,7 @@ import (
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store/engine"
)
@@ -15,9 +16,8 @@ import (
type DataStore struct {
engine.Interface
EditDuration time.Duration
Secret string
AdminStore admin.Store
MaxCommentSize int
Admins []string
// granular locks
scopedLocks struct {
@@ -31,6 +31,16 @@ const defaultCommentMaxSize = 2000
// Create prepares comment and forward to Interface.Create
func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
if comment, err = s.prepareNewComment(comment); err != nil {
return "", errors.Wrap(err, "failed to prepare comment")
}
return s.Interface.Create(comment)
}
// prepareNewComment sets new comment fields, hashing and sanitizing data
func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) {
// fill ID and time if empty
if comment.ID == "" {
comment.ID = uuid.New().String()
@@ -42,11 +52,14 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
if comment.Votes == nil {
comment.Votes = make(map[string]bool)
}
comment.Sanitize() // clear potentially dangerous js from all parts of comment
comment.Sanitize() // clear potentially dangerous js from all parts of comment
comment.User.HashIP(s.Secret) // replace ip by hash
return s.Interface.Create(comment)
secret, err := s.AdminStore.Key(comment.Locator.SiteID)
if err != nil {
return store.Comment{}, errors.Wrapf(err, "can't get secret for site %s", comment.Locator.SiteID)
}
comment.User.HashIP(secret) // replace ip by hash
return comment, nil
}
// SetPin pin/un-pin comment as special
@@ -109,6 +122,7 @@ type EditRequest struct {
Text string
Orig string
Summary string
Delete bool
}
// EditComment to edit text and update Edit info
@@ -123,6 +137,11 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
return comment, errors.Errorf("too late to edit %s", commentID)
}
if req.Delete { // delete request
comment.Deleted = true
return comment, s.Delete(locator, commentID, store.SoftDelete)
}
comment.Text = req.Text
comment.Orig = req.Orig
comment.Edit = &store.Edit{
@@ -165,9 +184,9 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
}
// IsAdmin checks if usesID in the list of admins
func (s *DataStore) IsAdmin(userID string) bool {
for _, admin := range s.Admins {
if admin == userID {
func (s *DataStore) IsAdmin(siteID string, userID string) bool {
for _, a := range s.AdminStore.Admins(siteID) {
if a == userID {
return true
}
}
backend/app/store/service/service_test.go
+31 -9
View File
@@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/admin"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine"
@@ -22,7 +23,8 @@ var testDb = "/tmp/test-remark.db"
func TestService_CreateFromEmpty(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
ks := admin.NewStaticKeyStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
@@ -45,7 +47,8 @@ func TestService_CreateFromEmpty(t *testing.T) {
func TestService_CreateFromPartial(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
ks := admin.NewStaticKeyStore("secret 123")
b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
@@ -70,7 +73,7 @@ func TestService_CreateFromPartial(t *testing.T) {
func TestService_Vote(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t)}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -115,7 +118,7 @@ func TestService_Vote(t *testing.T) {
func TestService_VoteAggressive(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t)}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -175,7 +178,7 @@ func TestService_VoteAggressive(t *testing.T) {
func TestService_VoteConcurrent(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t)}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
comment := store.Comment{
Text: "text",
@@ -206,7 +209,7 @@ func TestService_VoteConcurrent(t *testing.T) {
func TestService_Pin(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t)}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -230,7 +233,7 @@ func TestService_Pin(t *testing.T) {
func TestService_EditComment(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t)}
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -255,9 +258,28 @@ func TestService_EditComment(t *testing.T) {
assert.Nil(t, err, "allow second edit")
}
func TestService_DeleteComment(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
assert.Nil(t, res[0].Edit)
_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true})
assert.Nil(t, err)
c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
assert.Nil(t, err)
assert.True(t, c.Deleted)
t.Logf("%+v", c)
}
func TestService_EditCommentDurationFailed(t *testing.T) {
defer os.Remove(testDb)
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond}
b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
res, err := b.Last("radio-t", 0)
t.Logf("%+v", res[0])
@@ -274,7 +296,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
func TestService_ValidateComment(t *testing.T) {
b := DataStore{MaxCommentSize: 2000}
b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
longText := fmt.Sprintf("%4000s", "X")
tbl := []struct {
backend/app/store/user.go
+18 -16
View File
@@ -5,6 +5,7 @@ import (
"crypto/sha1"
"encoding/hex"
"fmt"
"hash"
"hash/crc64"
"io"
"log"
@@ -23,6 +24,7 @@ type User struct {
}
var reValidSha = regexp.MustCompile("^[a-fA-F0-9]{40}$")
var reValidCrc64 = regexp.MustCompile("^[a-fA-F0-9]{16}$")
// HashIP replace IP field with hashed hmac
func (u *User) HashIP(secret string) {
@@ -31,30 +33,30 @@ func (u *User) HashIP(secret string) {
// HashValue makes hmac with secret
func HashValue(val string, secret string) string {
if val == "" || reValidSha.MatchString(val) {
return val // already hashed or empty
}
key := []byte(secret)
h := hmac.New(sha1.New, key)
if _, err := io.WriteString(h, val); err != nil {
// fail back to crc64
log.Printf("[WARN] can't hash ip, %s", err)
return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA)))
}
return hex.EncodeToString(h.Sum(nil))
return hashWithFallback(hmac.New(sha1.New, key), val)
}
// EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
// we need hashing for parts of id, in some others hashing for non-User values.
func EncodeID(id string) string {
if reValidSha.MatchString(id) {
return id // already hashed or empty
return hashWithFallback(sha1.New(), id)
}
// hashWithFallback tries to has val with hash.Hash and fallback to crc if needed
func hashWithFallback(h hash.Hash, val string) string {
if reValidSha.MatchString(val) {
return val // already hashed or empty
}
h := sha1.New()
if _, err := io.WriteString(h, id); err != nil {
if _, err := io.WriteString(h, val); err != nil {
// fail back to crc64
log.Printf("[WARN] can't hash id %s, %s", id, err)
return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA)))
log.Printf("[WARN] can't hash id %s, %s", val, err)
if reValidCrc64.MatchString(val) {
return val // already crced
}
return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA)))
}
return hex.EncodeToString(h.Sum(nil))
}
backend/app/store/user_test.go
+23 -1
View File
@@ -1,6 +1,8 @@
package store
import (
"crypto/sha1"
"errors"
"testing"
"github.com/stretchr/testify/assert"
@@ -30,7 +32,7 @@ func TestUser_HashIP(t *testing.T) {
{"127.0.0.1", "ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741", "dbc7c999343f003f189f70aaf52cc04443f90790"},
{"8.8.8.8", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "70a46afce9633f010b06e129b8ad08243a1c4da9"},
{"8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2"},
{"", "", ""},
{"", "fbdb1d1b18aa6c08324b7d64b71fb76370690e1d", "823688dafca7393d24c871a2da98a84d8732e927"},
}
for i, tt := range tbl {
@@ -43,3 +45,23 @@ func TestUser_HashIP(t *testing.T) {
assert.Equal(t, tt.hash2, u.IP, "case #%d", i)
}
}
func TestUser_HashFailed(t *testing.T) {
r := hashWithFallback(mockHash{}, "123456789")
assert.Equal(t, "995dc9bbdf1939fa", r)
r = hashWithFallback(mockHash{}, "995dc9bbdf1939fa")
assert.Equal(t, "995dc9bbdf1939fa", r)
r = hashWithFallback(sha1.New(), "123456789")
assert.Equal(t, "f7c3bc1d808e04732adf679965ccc34ca7ae3441", r)
}
type mockHash struct{}
func (mock mockHash) Sum(b []byte) []byte { return nil }
func (mock mockHash) Reset() {}
func (mock mockHash) Size() int { return 0 }
func (mock mockHash) BlockSize() int { return 0 }
func (mock mockHash) Write(p []byte) (n int, err error) { return 0, errors.New("error") }
backend/remark.rest
+9 -15
View File
@@ -65,16 +65,16 @@ GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
### get comment by user id
GET {{host}}/api/v1/comments?site={{site}}&user=github_ef0f706a79cc24b17bbbb374cd234a691d034128&limit=5
GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
### get comment by user id2
GET {{host}}/api/v1/comments?site=radiot&user=github_0a4349d868946d7841424c9bdd4415629df771e6
GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
### get count
GET {{host}}/api/v1/count?site=remark&url={{url}}
GET {{host}}/api/v1/count?site={{site}}&url={{url}}
### get counts for many
POST {{host}}/api/v1/counts?site=remark
POST {{host}}/api/v1/counts?site={{site}}
Content-Type: application/json
[
@@ -86,14 +86,11 @@ Content-Type: application/json
### list commented posts
GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
### get config
GET {{host}}/api/v1/config
### block user
PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site=remark&block=1
PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1
### unblock user
PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site=remark&block=0
PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=0
### list blocked user
GET {{host}}/api/v1/admin/blocked?site={{site}}
@@ -105,19 +102,16 @@ DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
GET {{host}}/api/v1/info?site={{site}}&url=https://radio-t.com/p/2018/05/08/prep-597/
### post rss
GET {{host}}/api/v1/rss/post?site=={{site}}&url={{url}}
### reply rss
GET {{host}}/api/v1/rss/reply?site=radiot&user=github_ef0f706a79cc24b17bbbb374cd234a691d034128
GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
### site rss
PUT {{host}}/api/v1/rss/site?site=remark
GET {{host}}/api/v1/rss/site?site={{site}}
### get default avatar
GET {{host}}/api/v1/avatar/blah
### get config
GET {{host}}/api/v1/config?site=remark
GET {{host}}/api/v1/config?site={{site}}
### ping
GET {{host}}/ping
backend/scripts/backup.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -e
/srv/remark42 backup $@
backend/scripts/create-backup.sh
-9
View File
@@ -1,9 +0,0 @@
#!/bin/sh
# this scrips makes a backup file to /srv/var/userbackup-<site>-<timestamp>.gz
set -e
BACKUP_PATH=${BACKUP_PATH:-./var}
backup_file=${BACKUP_PATH}/userbackup-${1}-$(date +%s).gz
echo "make backup file for site $1 to $backup_file"
curl "http://127.0.0.1:8081/api/v1/admin/export?site=${1}&secret=${SECRET}" > ${backup_file}
echo "created backup ${backup_file}"
backend/scripts/import-disqus.sh
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
set -e
echo "import disqus file $1 to site $2"
curl -X POST -H "Content-Type: application/json" -d @/srv/var/$1 "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=disqus&secret=${SECRET}"
echo "import completed"

Some files were not shown because too many files have changed in this diff Show More