lint: typos

Rename "web"  to "frontend"

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
-/web/public/
+/frontend/node_modules/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,5 +19,5 @@ debug
 debug.test
 *.prof
 *.test
-/bin/
 remark42
+/backend/var/

.drone.yml

@@ -5,7 +5,7 @@ workspace:
 pipeline:
   
   build:
-    image: golang:1.10-alpine
+    image: golang:1.12-alpine
     commands:
       - sleep 5
       - nslookup mongo
@@ -99,4 +99,3 @@ services:
   mongo:
     image: mongo:3.6
     command: [ --smallfiles ]
-

.gitignore

@@ -8,13 +8,14 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
-/web/public/
-/web/coverage
+/frontend/node_modules/
+/frontend/public/
+/frontend/coverage
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
 .mongo
 remark42
-/bin/
+/bin/
+/backend/var/

Dockerfile

@@ -21,15 +21,17 @@ ARG DRONE_PULL_REQUEST
 ARG SKIP_BACKEND_TEST
 ARG MONGO_TEST
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD .git /go/src/github.com/umputun/remark/.git
+ADD backend /build/backend
+ADD .git /build/.git
+WORKDIR /build/backend
 
 # run tests
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
 RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
@@ -37,39 +39,34 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
-# coverage report
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    mkdir -p target && /script/coverage.sh ; \
-    else echo "skip backend coverage" ; fi
-
 # submit coverage to coverals if COVERALLS_TOKEN in env
 RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
     echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
-    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
+    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
@@ -77,9 +74,9 @@ ARG CI
 ARG SKIP_FRONTEND_TEST
 ARG NODE_ENV=production
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
     if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
@@ -95,8 +92,8 @@ ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 

Dockerfile.artifacts

@@ -1,22 +1,28 @@
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
+ARG DRONE
+ARG DRONE_TAG
+ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+
 ENV SKIP_FRONTEND_TEST=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
 ARG CI
 ARG NODE_ENV=production
 ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
@@ -30,7 +36,7 @@ WORKDIR /go/src/github.com/umputun/remark/backend
 ADD backend /go/src/github.com/umputun/remark/backend
 ADD README.md /go/src/github.com/umputun/remark/
 ADD LICENSE /go/src/github.com/umputun/remark/
-COPY --from=build-frontend /srv/web/public/ web
+COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
     export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \

README.md

@@ -3,19 +3,22 @@
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
 * Social login via Google, Facebook, Github and Yandex
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
 * Import from disqus and wordpress
-* Markdown support
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
+* Telegram notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
 * Self-contained executable can be deployed directly to Linux, Windows and MacOS
-* Clean, lightweight and fully customizable UI
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
@@ -25,10 +28,11 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 
   - [Install](#install)
     - [Backend](#backend)
-      - [With Docker](#with-docker) 
-      - [Without docker](#without-docker) 
+      - [With Docker](#with-docker)
+      - [Without docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
+      - [Quick installation test](#quick-installation-test)
       - [Register oauth2 providers](#register-oauth2-providers)
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
@@ -70,71 +74,83 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
+* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
 * pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
 #### Without docker
 
 * download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
-* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows) 
+* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
 * run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
 * alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
-| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                      |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -142,7 +158,7 @@ _this is the recommended way to run remark42_
 
 ##### Required parameters
 
-Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
+Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
 2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
@@ -165,9 +181,13 @@ services:
             - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
             - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
         volumes:
-            - ./var:/srv/var                        # persistent volume to store all remark42 data 
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
 ```
 
+#### Quick installation test
+
+To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
+
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
@@ -220,6 +240,13 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 
 For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
 
+##### Anonymous Auth Provider
+
+Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
+
+- name should be at least 3 characters long
+- name has to start from the letter and contains letters, numbers, underscores and spaces only.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
@@ -228,7 +255,7 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co
 
 #### Initial import from WordPress
 
-1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 
+1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
 2. Move this file to your remark42 host within `./var`
 3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
 
@@ -256,25 +283,25 @@ Restore will clean all comments first and then will processed with complete impo
 ##### Backup format
 
 Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
-unmarshaled from `Comment` struct (see below). 
+unmarshaled from `Comment` struct (see below).
 
 #### Admin users
 
-Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line. 
+Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line.
 
 ```
     environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
+        - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
-To get user id just login and click on your username or any other user you want to promote to admins. 
+To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
 ### Setup on your website
 
 #### Comments
 
-It's a main widget which renders list of comments. 
+It's a main widget which renders list of comments.
 
 Add this snippet to the bottom of web page:
 
@@ -300,7 +327,7 @@ And then add this node in the place where you want to see Remark42 widget:
 
 ```html
 <div id="remark42"></div>
-``` 
+```
 
 After that widget will be rendered inside this node.
 
@@ -309,13 +336,13 @@ After that widget will be rendered inside this node.
 Right now Remark has two themes: light and dark.
 You can pick one using configuration object,
 but there is also a possibility to switch between themes in runtime.
-For this purpose Remark adds to `window` object named `REMARK42`, 
+For this purpose Remark adds to `window` object named `REMARK42`,
 which contains function `changeTheme`.
 Just call this function and pass a name of the theme that you want to turn on:
 
 ```js
 window.REMARK42.changeTheme('light');
-```      
+```
 
 #### Last comments
 
@@ -326,7 +353,7 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
+    site_id: 'YOUR_SITE_ID',
   };
 
   (function() {
@@ -371,18 +398,18 @@ And then add a node like this in the place where you want to see a number of com
 <span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
 ```
 
-You can use as many nodes like this as you need to. 
-The script will found all them by the class `remark__counter`, 
+You can use as many nodes like this as you need to.
+The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
+Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
 
 ## Build from the source
 
 - to build docker container - `make docker`. This command will produce container `umputun/remark42`.
-- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable 
+- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
  `remark42` file with everything embedded.
- 
+
 ## Development
 
 You can use fully functional local version to develop and test both frontend & backend.
@@ -400,11 +427,11 @@ docker-compose -f compose-dev-frontend.yml up
 
 It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
 To access UI demo page go to `127.0.0.1:8080/web`.
-By default, you would be logged in as `dev_user` which defined as admin. 
+By default, you would be logged in as `dev_user` which defined as admin.
 You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
 
-Backend docker compose config by default skips running frontend related tests. 
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build. 
+Backend docker compose config by default skips running frontend related tests.
+Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
 
 ### Backend development
 
@@ -423,9 +450,9 @@ To run backend with mongodb store mongo container should be started first - `doc
 
 * install [Node.js 8](https://nodejs.org/en/) or higher;
 * install [NPM 6.1.0](https://www.npmjs.com/package/npm);
-* run `npm install` inside `./web`;
+* run `npm install` inside `./frontend`;
 * run `npm run build` there;
-* result files will be saved in `./web/public`.
+* result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
 It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
@@ -446,6 +473,10 @@ npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
+#### Frontend guide
+
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
 ## API
 
 ### Authorization
@@ -479,16 +510,24 @@ type Comment struct {
     User      User            `json:"user"`    // user info, read only
     Locator   Locator         `json:"locator"` // post locator
     Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
+    Vote      int             `json:"vote"`    // vote for the current user, -1/1/0.
+    Controversy float64       `json:"controversy,omitempty"` // comment controversy, read only
     Timestamp time.Time       `json:"time"`    // time stamp, read only
+    Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
+    PostTitle string          `json:"title"`   // post title
 }
 
 type Locator struct {
     SiteID string `json:"site"`     // site id
     URL    string `json:"url"`      // post url
 }
+
+type Edit struct {
+  Timestamp time.Time `json:"time" bson:"time"`
+  Summary   string    `json:"summary"`
+}
 ```
 
 * `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
@@ -515,11 +554,11 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
 ```go
- 	type EditRequest struct {
- 		Text    string `json:"text"`    // updated text
- 		Summary string `json:"summary"` // optional, summary of the edit
- 		Delete  bool   `json:"delete"`  // delete flag 
- 	}{}
+   type EditRequest struct {
+     Text    string `json:"text"`    // updated text
+     Summary string `json:"summary"` // optional, summary of the edit
+     Delete  bool   `json:"delete"`  // delete flag
+   }{}
 ```
 
 * `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
@@ -550,23 +589,36 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go
-  type config struct {
-      Version       string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
-      Admins        []string `json:"admins"`
-      Auth          []string `json:"auth_providers"`
-      LowScore      int      `json:"low_score"`
-      CriticalScore int      `json:"critical_score"`
+  type Config struct {
+      Version        string   `json:"version"`
+      EditDuration   int      `json:"edit_duration"`
+      MaxCommentSize int      `json:"max_comment_size"`
+      Admins         []string `json:"admins"`
+      AdminEmail     string   `json:"admin_email"`
+      Auth           []string `json:"auth_providers"`
+      LowScore       int      `json:"low_score"`
+      CriticalScore  int      `json:"critical_score"`
+      PositiveScore  bool     `json:"positive_score"`
+      ReadOnlyAge    int      `json:"readonly_age"`
+      MaxImageSize   int      `json:"max_image_size"`
   }
-  ``` 
+  ```
+
 * `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
-  
+
 ### RSS feeds
-  
+
 * `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -592,7 +644,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
 _all admin calls require auth and admin privilege_
 
-## Privacy 
+## Privacy
 
 * Remark42 is trying to be very sensitive to any private or semi-private information.
 * Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
@@ -603,7 +655,7 @@ _all admin calls require auth and admin privilege_
 * There are no third-party analytic services involved.
 * User can request all information remark42 knows about and export to gz file.
 * Supported complete cleanup of all information related to user's activity.
-* Cookie lifespan can be restricted to session-only. 
+* Cookie lifespan can be restricted to session-only.
 * All potentially sensitive data stored by remark42 hashed and encrypted.
 
 ## Technical details

backend/Gopkg.lock

@@ -1,429 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter",
-  ]
-  pruneopts = "UT"
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
-  name = "github.com/globalsign/mgo"
-  packages = [
-    ".",
-    "bson",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = "UT"
-  revision = "113d3961e7311526535a1ef7042196563d442761"
-  version = "r2018.06.15"
-
-[[projects]]
-  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware",
-  ]
-  pruneopts = "UT"
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
-  name = "github.com/go-chi/cors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dba6525398619dead495962a916728e7ee2ca322"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:5371050ba40cd7482fa2e9f0ff17fc18e2c37a6353ac012eaaacc6250eee4748"
-  name = "github.com/go-pkgz/auth"
-  packages = [
-    ".",
-    "avatar",
-    "logger",
-    "middleware",
-    "provider",
-    "token",
-  ]
-  pruneopts = "UT"
-  revision = "6f889bf1c6eb61c926dbb759897e6dc577b10655"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:1212e114344a5cdcc834ea69e19d456eef230f9784659080fee67e02ba2cb574"
-  name = "github.com/go-pkgz/mongo"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
-  name = "github.com/go-pkgz/repeater"
-  packages = [
-    ".",
-    "strategy",
-  ]
-  pruneopts = "UT"
-  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:e133aa7be09588b02198e4ddb98df5033b0319b56533881d0163ee51b903305b"
-  name = "github.com/go-pkgz/rest"
-  packages = [
-    ".",
-    "cache",
-    "logger",
-  ]
-  pruneopts = "UT"
-  revision = "553c0e1b55b215f8f55da4682ac57aff9aec8b6d"
-  version = "v1.1.5"
-
-[[projects]]
-  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = "UT"
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ce3f7860fd68bd2dd4c3735e2aed8c9de7c7d05bd6ad7d97a6bedcf4fe7b84fb"
-  name = "github.com/hashicorp/logutils"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "0dc08b1671f34c4250ce212759ebd880f743d883"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
-  name = "github.com/nullrocks/identicon"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7875f45b0022edded6377e40639d8aa620193a62"
-
-[[projects]]
-  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  pruneopts = "UT"
-  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
-  version = "v0.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
-  name = "golang.org/x/crypto"
-  packages = [
-    "acme",
-    "acme/autocert",
-  ]
-  pruneopts = "UT"
-  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64",
-  ]
-  pruneopts = "UT"
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom",
-  ]
-  pruneopts = "UT"
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex",
-  ]
-  pruneopts = "UT"
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = "UT"
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = "UT"
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/PuerkitoBio/goquery",
-    "github.com/coreos/bbolt",
-    "github.com/dgrijalva/jwt-go",
-    "github.com/didip/tollbooth",
-    "github.com/didip/tollbooth_chi",
-    "github.com/globalsign/mgo",
-    "github.com/globalsign/mgo/bson",
-    "github.com/go-chi/chi",
-    "github.com/go-chi/chi/middleware",
-    "github.com/go-chi/cors",
-    "github.com/go-chi/render",
-    "github.com/go-pkgz/auth",
-    "github.com/go-pkgz/auth/avatar",
-    "github.com/go-pkgz/auth/logger",
-    "github.com/go-pkgz/auth/provider",
-    "github.com/go-pkgz/auth/token",
-    "github.com/go-pkgz/mongo",
-    "github.com/go-pkgz/repeater",
-    "github.com/go-pkgz/rest",
-    "github.com/go-pkgz/rest/cache",
-    "github.com/go-pkgz/rest/logger",
-    "github.com/google/uuid",
-    "github.com/gorilla/feeds",
-    "github.com/hashicorp/go-multierror",
-    "github.com/hashicorp/logutils",
-    "github.com/jessevdk/go-flags",
-    "github.com/microcosm-cc/bluemonday",
-    "github.com/patrickmn/go-cache",
-    "github.com/pkg/errors",
-    "github.com/rakyll/statik/fs",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/acme/autocert",
-    "gopkg.in/russross/blackfriday.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/Gopkg.toml

@@ -1,48 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-
-[[constraint]]
-  name = "github.com/globalsign/mgo"
-  version = "r2018.06.15"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-
-
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"

backend/app/cmd/avatar.go

@@ -1,11 +1,11 @@
 package cmd
 
 import (
-	"log"
 	"path"
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth/avatar"
@@ -31,6 +31,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }

backend/app/cmd/backup.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 

backend/app/cmd/cleanup.go

@@ -3,13 +3,14 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"math"
 	"net/http"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -130,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL)
+	r, err := http.Get(listURL) //nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -157,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL)
+		r, err = http.Get(commentsURL) //nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}
@@ -180,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil

backend/app/cmd/cmd.go

@@ -5,7 +5,6 @@ package cmd
 import (
 	"bytes"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -13,6 +12,7 @@ import (
 	"text/template"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 
@@ -33,7 +33,7 @@ type CommonOpts struct {
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
-	c.RemarkURL = commonOpts.RemarkURL
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
 	c.SharedSecret = commonOpts.SharedSecret
 	c.Revision = commonOpts.Revision
 }

backend/app/cmd/import.go

@@ -6,12 +6,12 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 

backend/app/cmd/import_test.go

@@ -3,13 +3,13 @@ package cmd
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	flags "github.com/jessevdk/go-flags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

backend/app/cmd/restore.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"log"
 	"time"
+
+	log "github.com/go-pkgz/lgr"
 )
 
 // RestoreCommand set of flags and command for restore from backup

backend/app/cmd/server.go

@@ -3,22 +3,23 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
 	"path"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/auth/logger"
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/token"
 	"github.com/go-pkgz/mongo"
@@ -31,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -42,32 +44,37 @@ type ServerCommand struct {
 	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 
-	Sites          []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AdminPasswd    string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy     bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes       int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	LowScore       int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore  int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	ReadOnlyAge    int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration   time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	Port           int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot        string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation  string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles  int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	ImageProxy      bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
+	MaxCommentSize  int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
 
 	Auth struct {
 		TTL struct {
 			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
 			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
 		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Dev      bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
@@ -88,6 +95,22 @@ type StoreGroup struct {
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
 }
 
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+}
+
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
@@ -157,6 +180,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -170,16 +194,17 @@ func (s *ServerCommand) Execute(args []string) error {
 		stop := make(chan os.Signal, 1)
 		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
 		<-stop
-		log.Print("[WARN] interrupt signal")
+		log.Printf("[WARN] interrupt signal")
 		cancel()
 	}()
 
 	app, err := s.newServerApp()
 	if err != nil {
-		log.Fatalf("[ERROR] failed to setup application, %+v", err)
+		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
-		log.Printf("[WARN] remark terminated with error %+v", err)
+		log.Printf("[ERROR] remark terminated with error %+v", err)
 		return err
 	}
 	log.Printf("[INFO] remark terminated")
@@ -209,13 +234,21 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+
 	dataService := &service.DataStore{
-		Interface:      storeEngine,
-		EditDuration:   s.EditDuration,
-		AdminStore:     adminStore,
-		MaxCommentSize: s.MaxCommentSize,
-		MaxVotes:       s.MaxVotes,
-		TitleExtractor: service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		Interface:              storeEngine,
+		EditDuration:           s.EditDuration,
+		AdminStore:             adminStore,
+		MaxCommentSize:         s.MaxCommentSize,
+		MaxVotes:               s.MaxVotes,
+		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
+		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
 
 	loadingCache, err := s.makeCache()
@@ -268,15 +301,17 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		Cache:            loadingCache,
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
+		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     imageService,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+		da, errDevAuth := authenticator.DevAuth()
+		if errDevAuth != nil {
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -290,6 +325,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -315,12 +351,17 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -397,6 +438,27 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+				MaxHeight:  s.Image.ResizeHeight,
+				MaxWidth:   s.Image.ResizeWidth,
+			},
+			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
+		}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
@@ -467,10 +529,29 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 	if s.Auth.Dev {
+		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Anonymous {
+		log.Print("[INFO] anonymous access enabled")
+		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
+		authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
+			user = strings.TrimSpace(user)
+			if len(user) < 3 {
+				log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
+				return false, nil
+			}
+
+			if !isValidAnonName(user) {
+				log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
+				return false, nil
+			}
+			return true, nil
+		}))
+	}
+
 	if providers == 0 {
 		log.Printf("[WARN] no auth providers defined")
 	}
@@ -550,8 +631,28 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
-		Logger:            logger.Std,
+		Logger:            log.Default(),
+		RefreshCache:      newAuthRefreshCache(),
 	})
 	s.addAuthProviders(authenticator)
 	return authenticator
 }
+
+// authRefreshCache used by authenticator to minimize repeatable token refreshes
+type authRefreshCache struct {
+	*authcache.Cache
+}
+
+func newAuthRefreshCache() *authRefreshCache {
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
+}
+
+// Get implements cache getter with key converted to string
+func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
+	return c.Cache.Get(key.(string))
+}
+
+// Set implements cache setter with key converted to string
+func (c *authRefreshCache) Set(key, value interface{}) {
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
+}

backend/app/cmd/server_test.go

@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -16,6 +15,7 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/mongo"
 	flags "github.com/jessevdk/go-flags"
 
@@ -24,7 +24,7 @@ import (
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})
@@ -83,6 +83,48 @@ func TestServerApp_DevMode(t *testing.T) {
 	app.Wait()
 }
 
+func TestServerApp_AnonMode(t *testing.T) {
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18085
+		o.Auth.Anonymous = true
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+
+	// send ping
+	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	// try to login with good name
+	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+
+	// try to login with bad name
+	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	// try to login with short name
+	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	app.Wait()
+}
 func TestServerApp_WithMongo(t *testing.T) {
 
 	mongoURL := os.Getenv("MONGO_TEST")
@@ -102,15 +144,15 @@ func TestServerApp_WithMongo(t *testing.T) {
 		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
 	require.Nil(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation = "/tmp"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
 
 	// create app
 	app, err := opts.newServerApp()
 	require.Nil(t, err)
 
 	defer func() {
-		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, err)
+		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
+		assert.NoError(t, e)
 		conn := mongo.NewConnection(s, "test_remark", "")
 		_ = conn.WithDB(func(dbase *mgo.Database) error {
 			assert.NoError(t, dbase.DropDatabase())
@@ -145,8 +187,10 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
-		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
+		"--ssl.port=18443", "--image.fs.path=/tmp"})
 	require.Nil(t, err)
 
 	// create app
@@ -200,7 +244,7 @@ func TestServerApp_Failed(t *testing.T) {
 	p := flags.NewParser(&opts, flags.Default)
 
 	// RO bolt location
-	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
 	assert.Nil(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
@@ -264,7 +308,7 @@ func TestServerApp_MainSignal(t *testing.T) {
 
 	p := flags.NewParser(&s, flags.Default)
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
 	require.Nil(t, err)
@@ -316,7 +360,7 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})
@@ -345,7 +389,7 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
+	client := http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
@@ -377,7 +421,7 @@ func TestServerAuthHooks(t *testing.T) {
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	defer resp.Body.Close()
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "blocked user can't post")
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
 
 	app.Wait()
 }
@@ -390,7 +434,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
 	require.Nil(t, err)
-	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"

backend/app/main.go

@@ -2,13 +2,12 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"os/signal"
 	"runtime"
 	"syscall"
 
-	"github.com/hashicorp/logutils"
+	log "github.com/go-pkgz/lgr"
 	flags "github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
@@ -62,19 +61,11 @@ func main() {
 }
 
 func setupLog(dbg bool) {
-	filter := &logutils.LevelFilter{
-		Levels:   []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},
-		MinLevel: logutils.LogLevel("INFO"),
-		Writer:   os.Stdout,
-	}
-
-	log.SetFlags(log.Ldate | log.Ltime)
-
 	if dbg {
-		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
-		filter.MinLevel = logutils.LogLevel("DEBUG")
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
 	}
-	log.SetOutput(filter)
+	log.Setup(log.Msec, log.LevelBraces)
 }
 
 // getDump reads runtime stack and returns as a string

backend/app/main_test.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -11,19 +11,26 @@ import (
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(500 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		time.Sleep(5000 * time.Millisecond)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.Nil(t, e)
 	}()
 
 	wg := sync.WaitGroup{}
@@ -31,20 +38,29 @@ func TestMain(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
 		wg.Done()
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	var passed bool
+	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
+		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		if e != nil {
+			t.Logf("%+v", e)
+			return e
+		}
+		require.Nil(t, e)
+		defer resp.Body.Close()
+		assert.Equal(t, 200, resp.StatusCode)
+		body, e := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, e)
+		assert.Equal(t, "pong", string(body))
+		passed = true
+		return nil
+	})
 
-	// send ping
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
+	assert.NoError(t, err)
+	assert.Equal(t, true, passed, "at least on ping passed")
 
 	wg.Wait()
 }
@@ -54,5 +70,5 @@ func TestGetDump(t *testing.T) {
 	assert.True(t, strings.Contains(dump, "goroutine"))
 	assert.True(t, strings.Contains(dump, "[running]"))
 	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
-	log.Print("\n dump:" + dump)
+	log.Printf("\n dump: %s", dump)
 }

backend/app/migrator/backup.go

@@ -5,12 +5,12 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 

backend/app/migrator/backup_test.go

@@ -16,7 +16,8 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {

backend/app/migrator/disqus.go

@@ -3,10 +3,10 @@ package migrator
 import (
 	"encoding/xml"
 	"io"
-	"log"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue

backend/app/migrator/disqus_test.go

@@ -7,13 +7,13 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func TestDisqus_Import(t *testing.T) {

backend/app/migrator/migrator.go

@@ -5,9 +5,9 @@ package migrator
 
 import (
 	"io"
-	"log"
 	"os"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"

backend/app/migrator/native.go

@@ -2,23 +2,28 @@ package migrator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
-	"log"
+	"sync/atomic"
 
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/syncs"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
+const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
 // {"version": 1, comments:[{...}\n,{}], meta: {meta}}
 // each comments starts from the new line
 type Native struct {
-	DataStore Store
+	DataStore  Store
+	Concurrent int
 }
 
 type meta struct {
@@ -45,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
 		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -70,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -91,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -99,8 +104,13 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, err
 	}
 
-	failed := 0
-	total, comments := 0, 0
+	var failed, total, comments int64
+
+	concurrent := defaultConcurrent
+	if n.Concurrent > 0 {
+		concurrent = n.Concurrent
+	}
+	grp := syncs.NewSizedGroup(concurrent, syncs.Preemptive)
 
 	for {
 		comment := store.Comment{}
@@ -112,27 +122,34 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		total++
 
 		if err != nil {
+			atomic.AddInt64(&failed, 1)
 			failed++
 			continue
 		}
 
-		if _, err = n.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
+		// write comments in parallel
+		grp.Go(func(context.Context) {
+			if _, e := n.DataStore.Create(comment); e != nil {
+				atomic.AddInt64(&failed, 1)
+				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
+				return
+			}
+			num := atomic.AddInt64(&comments, 1)
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
+			}
+		})
+
 	}
 
+	grp.Wait()
+
 	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
+		return int(comments), errors.Errorf("failed to save %d comments", failed)
 	}
 	log.Printf("[INFO] imported %d comments from %d records", comments, total)
 
 	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
 
-	return comments, err
+	return int(comments), err
 }

backend/app/migrator/native_test.go

@@ -4,13 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

backend/app/migrator/wordpress.go

@@ -4,9 +4,9 @@ import (
 	"encoding/xml"
 	"html"
 	"io"
-	"log"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue
@@ -142,7 +143,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
 	assert.Nil(t, err, "create store")
 

backend/app/notify/notify.go

@@ -4,10 +4,11 @@ package notify
 import (
 	"context"
 	"fmt"
-	"log"
 	"sync"
 	"sync/atomic"
 
+	log "github.com/go-pkgz/lgr"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -28,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment

backend/app/notify/notify_test.go

@@ -4,14 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log"
 	"math/rand"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 

backend/app/notify/telegram.go

@@ -6,40 +6,46 @@ import (
 	"encoding/json"
 	"fmt"
 	"html"
-	"log"
 	"net/http"
-	"strings"
+	"strconv"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
 )
 
 // Telegram implements notify.Destination for telegram
 type Telegram struct {
-	channelName string
-	token       string
-	apiPrefix   string
-	timeout     time.Duration
+	channelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
+	token     string
+	apiPrefix string
+	timeout   time.Duration
 }
 
 const telegramTimeOut = 5000 * time.Millisecond
 const telegramAPIPrefix = "https://api.telegram.org/bot"
 
 // NewTelegram makes telegram bot for notifications
-func NewTelegram(token string, channelName string, timeout time.Duration, api string) (*Telegram, error) {
+func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
 
-	res := Telegram{channelName: channelName, token: token, apiPrefix: api, timeout: timeout}
-	res.channelName = strings.TrimPrefix(res.channelName, "@")
+	if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
+		channelID = "@" + channelID // if channelID not a number enforce @ prefix
+	}
+
+	res := Telegram{channelID: channelID, token: token, apiPrefix: api, timeout: timeout}
 	if res.apiPrefix == "" {
 		res.apiPrefix = telegramAPIPrefix
 	}
 	if res.timeout == 0 {
 		res.timeout = telegramTimeOut
 	}
-	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelName, res.timeout, res.timeout)
+	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
-	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	err := repeater.NewDefault(5, time.Millisecond*250).Do(ctx, func() error {
 		client := http.Client{Timeout: telegramTimeOut}
 		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
 		if err != nil {
@@ -81,7 +87,7 @@ func NewTelegram(token string, channelName string, timeout time.Duration, api st
 // Send to telegram channel
 func (t *Telegram) Send(ctx context.Context, req request) error {
 	client := http.Client{Timeout: telegramTimeOut}
-	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelName, req.comment.ID)
+	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
 
 	from := req.comment.User.Name
 	if req.comment.ParentID != "" {
@@ -92,8 +98,8 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	if req.comment.PostTitle != "" {
 		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
 	}
-	u := fmt.Sprintf("%s%s/sendMessage?chat_id=@%s&parse_mode=Markdown&disable_web_page_preview=true",
-		t.apiPrefix, t.token, t.channelName)
+	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
+		t.apiPrefix, t.token, t.channelID)
 
 	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
 	msg = html.UnescapeString(msg)
@@ -138,5 +144,5 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 }
 
 func (t *Telegram) String() string {
-	return "telegram: " + t.channelName
+	return "telegram: " + t.channelID
 }

backend/app/notify/telegram_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -20,6 +22,7 @@ func TestTelegram_New(t *testing.T) {
 	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NoError(t, err)
 	assert.NotNil(t, tb)
+	assert.Equal(t, "@remark_test", tb.channelID, "@ added")
 
 	st := time.Now()
 	_, err = NewTelegram("bad-resp", "remark_test", 2*time.Second, ts.URL+"/")
@@ -35,6 +38,17 @@ func TestTelegram_New(t *testing.T) {
 
 	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
 	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
+
+	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
+	assert.Error(t, err, "empty api url not allowed")
+
+	_, err = NewTelegram("good-token", "remark_test", 0, ts.URL+"/")
+	assert.NoError(t, err, "0 timeout allowed as default")
+
+	tb, err = NewTelegram("good-token", "1234567890", 2*time.Second, ts.URL+"/")
+	assert.NoError(t, err)
+	assert.NotNil(t, tb)
+	assert.Equal(t, "1234567890", tb.channelID, "no @ prefix")
 }
 
 func TestTelegram_Send(t *testing.T) {
@@ -49,10 +63,19 @@ func TestTelegram_Send(t *testing.T) {
 	cp := store.Comment{Text: "some parent text"}
 	cp.User.Name = "to"
 
+	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	assert.NoError(t, err)
+	c.PostTitle = "test title"
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
 	assert.NoError(t, err)
 
-	assert.Equal(t, "telegram: remark_test", tb.String())
+	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NotNil(t, err, "should failed")
+	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
+	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
+
+	assert.Equal(t, "telegram: @remark_test", tb.String())
 }
 
 func mockTelegramServer() *httptest.Server {

backend/app/rest/api/admin.go

@@ -2,15 +2,14 @@ package api
 
 import (
 	"errors"
-	"log"
 	"net/http"
 	"path"
 	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-
 	"github.com/go-pkgz/auth"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 
@@ -56,10 +55,10 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	err := a.dataService.Delete(locator, id, store.SoftDelete)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment", rest.ErrInternal)
 		return
 	}
-	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
@@ -72,7 +71,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
 	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -89,7 +88,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 
 	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
 	if err != nil || len(ucomments) == 0 {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
 	}
 	render.Status(r, http.StatusOK)
@@ -104,7 +103,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token", rest.ErrActionRejected)
 		return
 	}
 
@@ -112,19 +111,19 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
 	if !claims.User.BoolAttr("delete_me") {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token", rest.ErrNoAccess)
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+		avatarStore := a.authenticator.AvatarProxy().Store
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
 	}
@@ -148,7 +147,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -160,7 +159,7 @@ func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	users, err := a.dataService.Blocked(siteID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
 	}
 	render.JSON(w, r, users)
@@ -179,13 +178,14 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
 	if !roStatus {
 		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
+			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+				"read-only due the age", rest.ErrActionRejected)
 			return
 		}
 	}
 
 	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status", rest.ErrPostNotFound)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
@@ -199,7 +199,7 @@ func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
 
 	c, err := a.dataService.SetTitle(locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title", rest.ErrInternal)
 		return
 	}
 	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
@@ -216,7 +216,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	verifyStatus := r.URL.Query().Get("verified") == "1"
 
 	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
@@ -231,7 +231,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	pinStatus := r.URL.Query().Get("pin") == "1"
 
 	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))

backend/app/rest/api/admin_test.go

@@ -16,12 +16,13 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	R "github.com/go-pkgz/rest"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/go-pkgz/rest/cache"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestAdmin_Delete(t *testing.T) {
@@ -36,12 +37,33 @@ func TestAdmin_Delete(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
+	// check last comments
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments := []store.Comment{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+
+	// check multi count
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -52,6 +74,36 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
+
+	time.Sleep(250 * time.Millisecond)
+	// check last comments updated
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+
+	// check count updated
+	res, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah")
+	assert.Equal(t, 200, code)
+	b := map[string]interface{}{}
+	err = json.Unmarshal([]byte(res), &b)
+	assert.Nil(t, err)
+	t.Logf("%#v", b)
+	assert.Equal(t, 1.0, b["count"], "should report 1 comments")
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestAdmin_Title(t *testing.T) {
@@ -61,11 +113,13 @@ func TestAdmin_Title(t *testing.T) {
 	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -262,6 +316,7 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.True(t, comments.Comments[0].Deleted)
 
+	srv.Cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
@@ -392,6 +447,33 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
+func TestAdmin_ReadOnlyNoComments(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	client := http.Client{}
+
+	// set post to read-only
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	assert.NotNil(t, err)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&format=tree")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
+}
+
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -640,7 +722,7 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	req.SetBasicAuth("admin", "bad-password")
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
+	assert.Equal(t, 403, resp.StatusCode)
 
 	// try bad user
 	badClaims := claims

backend/app/rest/api/migrator.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"sync"
@@ -14,7 +13,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
@@ -57,13 +56,14 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	tmpfile, err := m.saveTemp(r.Body)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -79,25 +79,26 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	if err := r.ParseMultipartForm(20 * 1024 * 1024); err != nil { // 20M max memory, if bigger will make a file
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
 		return
 	}
 
 	file, _, err := r.FormFile("file")
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import from the request")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
 	tmpfile, err := m.saveTemp(file)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -156,7 +157,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
 		return
 	}
 }

backend/app/rest/api/migrator_test.go

@@ -3,10 +3,10 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
@@ -15,26 +15,23 @@ import (
 	"testing"
 	"time"
 
-	"github.com/go-pkgz/auth/token"
-
-	bolt "github.com/coreos/bbolt"
+	"github.com/coreos/bbolt"
 	"github.com/go-chi/chi"
 	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/auth/token"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -51,19 +48,12 @@ func TestMigrator_Import(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -86,18 +76,12 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 }
+
 func TestMigrator_ImportFromWP(t *testing.T) {
-	srv, ds, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
@@ -114,34 +98,12 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.NoError(t, ds.Interface.Close())
-
-	tsAccess, _, teardownAccess := startupT(t)
-	defer teardownAccess()
-
-	res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
-	require.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-	t.Logf("%+v", comments)
-	assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
-		comments[0].Text)
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -155,13 +117,12 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
@@ -180,17 +141,16 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
@@ -209,12 +169,13 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, teardown := prepImportSrv(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -227,13 +188,7 @@ func TestMigrator_Export(t *testing.T) {
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 
 	// check file mode
 	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
@@ -274,12 +229,27 @@ func TestMigrator_Export(t *testing.T) {
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
+func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
+}
+
+func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
+	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
 	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc = &Migrator{
+	svc := &Migrator{
 		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
 		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 		NativeImporter:    &migrator.Native{DataStore: dataStore},
@@ -297,12 +267,12 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http
 	am := a.Middleware()
 	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
 	ts = httptest.NewServer(routes)
-	return svc, dataStore, ts
-}
-
-func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
-	ts.Close()
-	_ = os.Remove(testDb)
+	teardown = func() {
+		ts.Close()
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
+	return ts, teardown
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"path"
 	"strings"
@@ -19,18 +18,19 @@ import (
 	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/cors"
 	"github.com/go-chi/render"
-	"github.com/pkg/errors"
-	"github.com/rakyll/statik/fs"
-
 	"github.com/go-pkgz/auth"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/go-pkgz/rest/logger"
+	"github.com/pkg/errors"
+	"github.com/rakyll/statik/fs"
 
 	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -45,6 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     *image.Service
 
 	WebRoot         string
 	RemarkURL       string
@@ -54,6 +55,7 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
+	UpdateLimiter float64
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
@@ -80,6 +82,7 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		err := s.httpServer.ListenAndServe()
@@ -89,7 +92,10 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		go func() {
@@ -106,7 +112,11 @@ func (s *Rest) Run(port int) {
 		m := s.makeAutocertManager()
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.lock.Unlock()
 
 		go func() {
@@ -156,7 +166,7 @@ func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 
 func (s *Rest) routes() chi.Router {
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer)
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
 	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
 	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
 
@@ -183,13 +193,13 @@ func (s *Rest) routes() chi.Router {
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Flags(logger.All), logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
+		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
+		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
-		r.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
 
@@ -199,7 +209,8 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
-			rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
 		})
 
@@ -207,7 +218,9 @@ func (s *Rest) routes() chi.Router {
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
 			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Flags(logger.All), logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Use(middleware.NoCache)
+			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
 			ropen.Get("/find", s.findCommentsCtrl)
 			ropen.Get("/id/{id}", s.commentByIDCtrl)
 			ropen.Get("/comments", s.findUserCommentsCtrl)
@@ -218,32 +231,69 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
-
-			ropen.Mount("/rss", s.rssRoutes())
 			ropen.Mount("/img", s.ImageProxy.Routes())
+			ropen.Mount("/rss", s.rssRoutes())
+		})
+
+		// open routes, cached
+		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(authMiddleware.Trace)
+			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
 			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
+			rauth.Use(middleware.NoCache)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
 			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
 			rauth.Get("/userdata", s.userAllDataCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
 
 			// admin routes, admin users only
 			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
 		})
+
+		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(middleware.NoCache)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+
+			rauth.Put("/comment/{id}", s.updateCommentCtrl)
+			rauth.Post("/comment", s.createCommentCtrl)
+			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
+		})
+
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+		})
+
 	})
 
 	// respond to /robots.txt with the list of allowed paths
 	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
 		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+				"/img", "/avatar", "/picture"}
 			for i := range allowed {
 				allowed[i] = "Allow: /api/v1" + allowed[i]
 			}
@@ -266,20 +316,54 @@ func (s *Rest) routes() chi.Router {
 	return router
 }
 
+func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+
+	res = s.adminService.alterComments(comments, r) // apply admin's alteration
+
+	// prepare vote info for client view
+	vote := func(c store.Comment, r *http.Request) store.Comment {
+
+		c.Vote = 0 // default is "none" (not voted)
+
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			c.Votes = nil // hide voters list and don't set Vote for non-authed user
+			return c
+		}
+
+		if v, ok := c.Votes[user.ID]; ok {
+			if v {
+				c.Vote = 1
+			} else {
+				c.Vote = -1
+			}
+		}
+
+		c.Votes = nil // hide voters list
+		return c
+	}
+
+	for i, c := range res {
+		c = vote(c, r)
+		res[i] = c
+	}
+
+	return res
+}
+
 // serves static files from /web or embedded by statik
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 
 	var webFS http.Handler
 
 	statikFS, err := fs.New()
-	if err == nil {
-		log.Printf("[INFO] run file server for %s, embedded", root)
-		webFS = http.FileServer(statikFS)
-	}
 	if err != nil {
 		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
 		log.Printf("[INFO] run file server for %s, path %s", root, path)
 		webFS = http.FileServer(root)
+	} else {
+		log.Printf("[INFO] run file server for %s, embedded", root)
+		webFS = http.FileServer(statikFS)
 	}
 
 	origPath := path
@@ -311,7 +395,8 @@ func encodeJSONWithHTML(v interface{}) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
+func filterComments(comments []store.Comment, fn func(c store.Comment) bool) []store.Comment {
+	filtered := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			filtered = append(filtered, c)
@@ -324,9 +409,42 @@ func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (fi
 // admins will have different keys in order to prevent leak of admin-only data to regular users
 func URLKey(r *http.Request) string {
 	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
-		key = adminPrefix + key
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil && user.Admin {
+		key = adminPrefix + key // make separate cache key for admins
 	}
 	return key
 }
+
+// URLKeyWithUser gets url from request to use it as cache key and attaching user ID
+// admins will have different keys in order to prevent leak of admin-only data to regular users
+func URLKeyWithUser(r *http.Request) string {
+	adminPrefix := "admin!!"
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil {
+		if user.Admin {
+			key = adminPrefix + user.ID + "!!" + key // make separate cache key for admins
+		} else {
+			key = user.ID + "!!" + key // make separate cache key for authed users
+		}
+	}
+	return key
+}
+
+// rejectAnonUser is a middleware rejecting anonymous users
+func rejectAnonUser(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		if strings.HasPrefix(user.ID, "anonymous_") {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}

backend/app/rest/api/rest_private.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"strings"
 	"time"
@@ -13,11 +12,11 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	multierror "github.com/hashicorp/go-multierror"
-
 	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
+	multierror "github.com/hashicorp/go-multierror"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
@@ -29,7 +28,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -41,32 +40,36 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment.Orig = comment.Text // original comment text, prior to md render
 	if err := s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	comment = s.CommentFormatter.Format(comment)
 
 	// check if user blocked
 	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	if s.isReadOnly(comment.Locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	id, err := s.DataService.Create(comment)
+	if err == service.ErrRestrictedWordsFound {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
+		return
+	}
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment", rest.ErrInternal)
 		return
 	}
 
 	// DataService modifies comment
 	finalComment, err := s.DataService.Get(comment.Locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
@@ -92,7 +95,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -105,12 +108,13 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	var currComment store.Comment
 	var err error
 	if currComment, err = s.DataService.Get(locator, id); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
 
 	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+			"can not edit comments for other users", rest.ErrNoAccess)
 		return
 	}
 
@@ -122,12 +126,18 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res, err := s.DataService.EditComment(locator, id, editReq)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
+	if err == service.ErrRestrictedWordsFound {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID))
+	if err != nil {
+		code := s.parseError(err, rest.ErrCommentRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
+		return
+	}
+
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
@@ -151,19 +161,20 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	vote := r.URL.Query().Get("vote") == "1"
 
 	if s.isReadOnly(locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	// check if user blocked
 	if s.adminService.checkBlocked(locator.SiteID, user) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
+		code := s.parseError(err, rest.ErrVoteRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
@@ -176,7 +187,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info", rest.ErrInternal)
 		return
 	}
 
@@ -202,14 +213,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
+		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
+		b, errUser := json.Marshal(comments)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
@@ -221,7 +232,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	merr = multierror.Append(merr, write([]byte(`}`)))
 	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info", rest.ErrInternal)
 		return
 	}
 
@@ -251,7 +262,7 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 
 	tokenStr, err := s.Authenticator.TokenService().Token(claims)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
@@ -259,6 +270,31 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
+// POST /image - save image with form request
+func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"id": id})
+}
+
 func (s *Rest) isReadOnly(locator store.Locator) bool {
 	if s.ReadOnlyAge > 0 {
 		// check RO by age
@@ -268,3 +304,28 @@ func (s *Rest) isReadOnly(locator store.Locator) bool {
 	}
 	return s.DataService.IsReadOnly(locator) // ro manually
 }
+
+func (s *Rest) parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private_test.go

@@ -1,22 +1,37 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/rest"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
 func TestRest_Create(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -101,6 +116,24 @@ func TestRest_CreateTooBig(t *testing.T) {
 	assert.Equal(t, "can't bind comment", c["details"])
 }
 
+func TestRest_CreateWithRestrictedWord(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", badComment)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.Nil(t, err)
+	assert.Equal(t, "comment contains restricted words", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_CreateRejected(t *testing.T) {
 
 	ts, _, teardown := startupT(t)
@@ -198,21 +231,34 @@ func TestRest_UpdateDelete(t *testing.T) {
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
+	// check multi count updated
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, b.StatusCode, string(body))
 
 	// comments returned by update
 	c2 := store.Comment{}
 	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, id, c2.ID)
 	assert.True(t, c2.Deleted)
 
@@ -225,6 +271,18 @@ func TestRest_UpdateDelete(t *testing.T) {
 	assert.Equal(t, "", c3.Text)
 	assert.Equal(t, "", c3.Orig)
 	assert.True(t, c3.Deleted)
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestRest_UpdateNotOwner(t *testing.T) {
@@ -246,7 +304,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	body, err := ioutil.ReadAll(b.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
+	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
 	client = http.Client{}
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
@@ -258,6 +316,31 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
 }
 
+func TestRest_UpdateWithRestrictedWords(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "What the quack is that?", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id := addComment(t, c1, ts)
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"text":"What the duck is that?", "summary":"my edit"}`))
+	assert.Nil(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err := client.Do(req)
+	assert.Nil(t, err)
+	body, err := ioutil.ReadAll(b.Body)
+	assert.Nil(t, err)
+	c := R.JSON{}
+	err = json.Unmarshal(body, &c)
+	assert.Nil(t, err)
+	assert.Equal(t, 400, b.StatusCode, string(body))
+	assert.Equal(t, "comment contains restricted words", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_Vote(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -275,7 +358,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("admin", "password")
+		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -283,22 +366,65 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, 200, vote(1), "first vote allowed")
 	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"admin": true}, cr.Votes)
+	assert.Equal(t, 1, cr.Vote)
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
+	assert.Equal(t, 0, cr.Vote)
+
+	assert.Equal(t, 200, vote(-1), "opposite vote allowed one more time")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	assert.Equal(t, 400, vote(-1), "dbl vote not allowed")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	req, err := http.NewRequest("GET",
+		fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+	assert.NoError(t, err)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	cr = store.Comment{}
+	err = json.NewDecoder(resp.Body).Decode(&cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -364,13 +490,12 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
 
-	for i := 0; i < 478; i++ {
+	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
 		c.Timestamp = c.Timestamp.Add(time.Second)
 		_, err := srv.DataService.Create(c)
 		require.Nil(t, err)
 	}
-
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
@@ -386,7 +511,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
 		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
+	assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
@@ -421,3 +546,163 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	savePic := func(name string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 1462, len(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+}
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	svc.ImageService = &image.Service{
+		Store: &image.FileSystem{
+			Staging:  "/tmp/remark42/images.staging",
+			Location: "/tmp/remark42/images",
+			MaxSize:  2000,
+		},
+		TTL: time.Millisecond * 100,
+	}
+	svc.DataService.EditDuration = time.Millisecond * 100
+	svc.DataService.ImageService = svc.ImageService
+
+	uploadPicture := func(file string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.Contains(t, m["id"], ".png")
+		t.Logf(string(body))
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png")
+	id2 := uploadPicture("pic2.png")
+	id3 := uploadPicture("pic3.png")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	svc := Rest{}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := svc.parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}

backend/app/rest/api/rest_public.go

@@ -1,16 +1,16 @@
 package api
 
 import (
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/base64"
-	"log"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 
@@ -18,7 +18,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
 // find comments for given post. Returns in tree or plain formats, sorted
 func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -28,17 +28,20 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
 
-	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Find(locator, sort)
 		if e != nil {
-			return nil, e
+			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.adminService.alterComments(comments, r)
+		maskedComments := s.alterComments(comments, r)
 		var b []byte
 		switch r.URL.Query().Get("format") {
 		case "tree":
 			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			if tree.Nodes == nil { // eliminate json nil serialization
+				tree.Nodes = []*rest.Node{}
+			}
 			if s.DataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
@@ -54,7 +57,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -67,19 +70,19 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
 	user, err := rest.GetUserInfo(r)
 	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info", rest.ErrNoAccess)
 		return
 	}
 	comment.User = user
 	comment.Orig = comment.Text
 	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
@@ -102,7 +105,7 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info", rest.ErrPostNotFound)
 		return
 	}
 
@@ -127,14 +130,14 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments", rest.ErrInternal)
 		return
 	}
 
@@ -154,10 +157,10 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
+	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -183,13 +186,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
-	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID)
+	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.User(siteID, userID, limit, 0)
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		count, e := s.DataService.UserCount(siteID, userID)
 		if e != nil {
@@ -200,7 +203,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -222,7 +225,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		Auth           []string `json:"auth_providers"`
 		LowScore       int      `json:"low_score"`
 		CriticalScore  int      `json:"critical_score"`
+		PositiveScore  bool     `json:"positive_score"`
 		ReadOnlyAge    int      `json:"readonly_age"`
+		MaxImageSize   int      `json:"max_image_size"`
 	}
 
 	cnf := config{
@@ -233,7 +238,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		AdminEmail:     s.DataService.AdminStore.Email(siteID),
 		LowScore:       s.ScoreThresholds.Low,
 		CriticalScore:  s.ScoreThresholds.Critical,
+		PositiveScore:  s.DataService.PositiveScore,
 		ReadOnlyAge:    s.ReadOnlyAge,
+		MaxImageSize:   s.ImageService.Store.SizeLimit(),
 	}
 
 	cnf.Auth = []string{}
@@ -253,29 +260,26 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	count, err := s.DataService.Count(locator)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
 	}
 	render.JSON(w, r, R.JSON{"count": count, "locator": locator})
 }
 
-// POST /count?site=siteID - get number of comments for posts from post body
+// POST /counts?site=siteID - get number of comments for posts from post body
 func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
 		return
 	}
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
-	if _, err := hasher.Write([]byte(k)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	h := sha1.Sum([]byte(k)) //nolint
+	sha := base64.URLEncoding.EncodeToString(h[:])
+
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
@@ -286,7 +290,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -318,7 +322,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -326,3 +330,46 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't render posts lits for site %s", siteID)
 	}
 }
+
+// GET /picture/{user}/{id} - get picture
+func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
+	imgRdr, size, err := s.ImageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer imgRdr.Close()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}

backend/app/rest/api/rest_public_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -69,8 +70,12 @@ func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -83,10 +88,10 @@ func TestRest_Find(t *testing.T) {
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
@@ -194,6 +199,10 @@ func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "[]\n", res, "empty last should return empty list")
+
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
@@ -204,7 +213,7 @@ func TestRest_Last(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	id2 := addComment(t, c2, ts)
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
@@ -227,6 +236,7 @@ func TestRest_Last(t *testing.T) {
 
 	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
 	assert.Nil(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
@@ -367,6 +377,37 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 3, pi[1].Count)
 }
 
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
+}
+
 func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -382,7 +423,9 @@ func TestRest_Config(t *testing.T) {
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
 	assert.Equal(t, -10., j["critical_score"])
+	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
+	assert.Equal(t, 10000., j["max_image_size"])
 	t.Logf("%+v", j)
 }
 
@@ -431,5 +474,5 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }

backend/app/rest/api/rest_test.go

@@ -4,10 +4,13 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -22,19 +25,22 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
 var testHTML = "/tmp/test-remark.html"
 var getStartedHTML = "/tmp/getstarted.html"
 
 var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -175,18 +181,98 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 	srv.Shutdown()
 }
 
+func TestRest_rejectAnonUser(t *testing.T) {
+
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello")
+	}))))
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
+
+	resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
+
+	resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
+}
+
+func Test_URLKey(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKey(r))
+		})
+	}
+
+}
+
+func Test_URLKeyWithUser(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "user!!http://example.com/1"},
+		{"http://example.com/2", store.User{ID: "user2"}, "user2!!http://example.com/2"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!user!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKeyWithUser(r))
+		})
+	}
+
+}
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+
+	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
+	os.Remove(testDb)
+	os.Remove(testHTML)
+	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
+
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 
+	memCache, err := cache.NewMemoryCache()
+	assert.NoError(t, err)
+
 	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
-		Interface:      b,
-		EditDuration:   5 * time.Minute,
-		MaxCommentSize: 4000,
-		AdminStore:     adminStore,
-		MaxVotes:       service.UnlimitedVotes,
+		Interface:              b,
+		EditDuration:           5 * time.Minute,
+		MaxCommentSize:         4000,
+		AdminStore:             adminStore,
+		MaxVotes:               service.UnlimitedVotes,
+		RestrictedWordsMatcher: restrictedWordsMatcher,
 	}
 
 	srv = &Rest{
@@ -196,10 +282,18 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
 		}),
-		Cache:     &cache.Nop{},
+		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+				Staging:    "/tmp/pics-remark42/staging",
+			},
+			TTL: time.Millisecond * 100,
+		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -221,15 +315,30 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 
 	teardown = func() {
 		ts.Close()
-		srv.DataService.Close()
+		require.NoError(t, srv.DataService.Close())
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
+// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+func fakeAuth(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("fake_id") != "" {
+			r = rest.SetUserInfo(r, store.User{
+				ID:   r.URL.Query().Get("fake_id"),
+				Name: r.URL.Query().Get("fake_name"),
+			})
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
 	require.Nil(t, err)
@@ -239,6 +348,14 @@ func get(t *testing.T, url string) (string, int) {
 	return string(body), r.StatusCode
 }
 
+func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+	if token != "" {
+		r.Header.Set("X-JWT", token)
+	}
+	return client.Do(r)
+}
+
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
@@ -292,10 +409,3 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
-
-func cleanup(ts *httptest.Server, srv *Rest) {
-	ts.Close()
-	srv.DataService.Close()
-	os.Remove(testDb)
-	os.Remove(testHTML)
-}

backend/app/rest/api/rss.go

@@ -2,23 +2,22 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"time"
 
 	"github.com/go-chi/chi"
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
 	"github.com/pkg/errors"
 
-	"github.com/go-pkgz/rest/cache"
-
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
 )
 
 const maxRssItems = 20
-const maxLastCommentsReply = 1000
-const maxReplyDuration = 24 * time.Hour
+const maxLastCommentsReply = 5000
+const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
@@ -42,8 +41,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments)
+		comments = s.alterComments(comments, r)
+		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
@@ -51,14 +50,14 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrPostNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -74,9 +73,9 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
+		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
@@ -84,13 +83,13 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -101,18 +100,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
+	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
 	data, err := s.Cache.Get(key, func() (res []byte, e error) {
 		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		replies := []store.Comment{}
 		for _, c := range comments {
 			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
 				break
 			}
+			if c.User.ID != userID {
+				userName = c.User.Name
+			}
 			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
 				var pc store.Comment
 				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
@@ -124,7 +127,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		rss, e := s.toRssFeed(siteID, replies)
+		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
@@ -132,19 +135,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
+func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
+	if description == "" {
+		description = "comment updates"
+	}
 	lastCommentTS := time.Unix(0, 0)
 	if len(comments) > 0 {
 		lastCommentTS = comments[0].Timestamp
@@ -153,7 +159,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 	feed := &feeds.Feed{
 		Title:       "Remark42 comments",
 		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
+		Description: description,
 		Created:     lastCommentTS,
 	}
 
@@ -165,6 +171,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 			Description: c.Text,
 			Created:     c.Timestamp,
 			Author:      &feeds.Author{Name: c.User.Name},
+			Id:          c.ID,
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment

backend/app/rest/api/rss_test.go

@@ -8,20 +8,26 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestServer_RssPost(t *testing.T) {
-	ts, _, teardown := startupT(t)
+	ts, rst, teardown := startupT(t)
 	defer teardown()
 
 	waitOnSecChange()
 
 	c1 := store.Comment{
+		ID:      "1234567890",
 		Text:    "test 123",
 		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
 	}
-	id1 := addComment(t, c1, ts)
+	id1, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	assert.Equal(t, "1234567890", id1)
 	pubDate := time.Now().Format(time.RFC1123Z)
 
 	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
@@ -32,17 +38,18 @@ func TestServer_RssPost(t *testing.T) {
         <channel>
             <title>Remark42 comments</title>
             <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
+            <description>post comments for https://radio-t.com/blah1</description>
             <pubDate>%s</pubDate>
             <item>
 		      <title>developer one</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
+		      <link>https://radio-t.com/blah1#remark42__comment-1234567890</link>
+		      <description>test 123</description>
 		      <author>developer one</author>
+              <guid>1234567890</guid>
 		      <pubDate>%s</pubDate>
             </item>
          </channel>
-	</rss>`, pubDate, id1, pubDate)
+	</rss>`, pubDate, pubDate)
 
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
@@ -52,7 +59,7 @@ func TestServer_RssPost(t *testing.T) {
 }
 
 func TestServer_RssSite(t *testing.T) {
-	ts, _, teardown := startupT(t)
+	ts, rst, teardown := startupT(t)
 	defer teardown()
 
 	waitOnSecChange()
@@ -60,16 +67,24 @@ func TestServer_RssSite(t *testing.T) {
 	pubDate := time.Now().Format(time.RFC1123Z)
 
 	c1 := store.Comment{
+		ID:      "comment-id-1",
 		Text:    "test 123",
 		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
+		ID:      "comment-id-2",
 		Text:    "xyz test",
 		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
 	}
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
 
+	_, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = rst.DataService.Create(c2)
+	require.NoError(t, err)
+
+	require.NoError(t, err)
 	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
 	assert.Equal(t, 200, code)
 	t.Log(res)
@@ -78,24 +93,26 @@ func TestServer_RssSite(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>radio-t</link>
-		    <description>comment updates</description>
+		    <description>site comment for radio-t</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
-		      <link>https://radio-t.com/blah11#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
+		      <link>https://radio-t.com/blah11#remark42__comment-comment-id-2</link>
+		      <description>xyz test</description>
 		      <author>developer one</author>
+              <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
 		    </item>
 		    <item>
 		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-1</link>
+		      <description>test 123</description>
 		      <author>developer one</author>
+              <guid>comment-id-1</guid>
 		      <pubDate>%s</pubDate>
 		    </item>
 		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
+		</rss>`, pubDate, pubDate, pubDate)
 
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
@@ -105,7 +122,7 @@ func TestServer_RssSite(t *testing.T) {
 }
 
 func TestServer_RssWithReply(t *testing.T) {
-	ts, _, teardown := startupT(t)
+	ts, rst, teardown := startupT(t)
 	defer teardown()
 
 	waitOnSecChange()
@@ -113,16 +130,23 @@ func TestServer_RssWithReply(t *testing.T) {
 	pubDate := time.Now().Format(time.RFC1123Z)
 
 	c1 := store.Comment{
+		ID:      "comment-id-1",
 		Text:    "test 123",
 		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
 	}
 	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		ID:       "comment-id-2",
+		ParentID: "comment-id-1",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u1", Name: "developer one"},
 	}
-	id1 := addComment(t, c1, ts)
-	c2.ParentID = id1
-	id2 := addComment(t, c2, ts)
+
+	_, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = rst.DataService.Create(c2)
+	require.NoError(t, err)
 
 	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
 	assert.Equal(t, 200, code)
@@ -132,24 +156,26 @@ func TestServer_RssWithReply(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
+		    <description>post comments for https://radio-t.com/blah10</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one &gt; developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
+		      <description>xyz test</description>
 		      <author>developer one</author>
+              <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
 		    </item>
 		    <item>
 		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-1</link>
+		      <description>test 123</description>
 		      <author>developer one</author>
+              <guid>comment-id-1</guid>
 		      <pubDate>%s</pubDate>
 		    </item>
 		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
+		</rss>`, pubDate, pubDate, pubDate)
 
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
@@ -164,39 +190,49 @@ func TestServer_RssReplies(t *testing.T) {
 	pubDate := time.Now().Format(time.RFC1123Z)
 
 	c1 := store.Comment{
+		ID:      "comment-1",
 		Text:    "c1",
 		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
 		User:    store.User{ID: "user1", Name: "user1"},
 	}
-	id1, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
 	c2 := store.Comment{
+		ID:       "comment-2",
 		Text:     "reply to c1 from user2",
-		ParentID: id1,
+		ParentID: "comment-1",
 		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
 		User:     store.User{ID: "user2", Name: "user2"},
 	}
-	id2, err := srv.DataService.Create(c2)
-	assert.Nil(t, err)
 	c3 := store.Comment{
+		ID:       "comment-3",
 		Text:     "reply to c1 from user3",
-		ParentID: id1,
+		ParentID: "comment-1",
 		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
 		User:     store.User{ID: "user3", Name: "user3"},
 	}
-	id3, err := srv.DataService.Create(c3)
-	assert.Nil(t, err)
 	c4 := store.Comment{
+		ID:       "comment-4",
 		Text:     "reply to c2 from developer one",
-		ParentID: id2,
+		ParentID: "comment-2",
 		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:     store.User{ID: "dev", Name: "developer one"},
 	}
-	addComment(t, c4, ts)
 	c5 := store.Comment{
+		ID:      "comment-5",
 		Text:    "developer one",
 		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:    store.User{ID: "dev", Name: "developer one"},
 	}
-	addComment(t, c5, ts)
+
+	_, err := srv.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c2)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c3)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c4)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c5)
+	require.NoError(t, err)
 
 	// replies to c1 (user1). Must be [c3, c2]
 	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t")
@@ -206,24 +242,26 @@ func TestServer_RssReplies(t *testing.T) {
 	    <channel>
 	        <title>Remark42 comments</title>
 	        <link>radio-t</link>
-	        <description>comment updates</description>
+	        <description>replies to user2</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
+		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
 		      <description>reply to c1 from user3</description>
 		      <author>user3</author>
+              <guid>comment-3</guid> 
 		      <pubDate>%s</pubDate>
 			</item>
 			<item>
 		      <title>user2 &gt; user1</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
+		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
 		      <description>reply to c1 from user2</description>
 		      <author>user2</author>
+              <guid>comment-2</guid> 
 		      <pubDate>%s</pubDate>
 			</item>
 	     </channel>
-	</rss>`, pubDate, id3, pubDate, id2, pubDate)
+	</rss>`, pubDate, pubDate, pubDate)
 	expected, res = cleanRssFormatting(expected, res)
 	assert.Equal(t, expected, res)
 

backend/app/rest/api/ssl.go

@@ -2,13 +2,13 @@ package api
 
 import (
 	"crypto/tls"
-	"log"
 	"net/http"
 	"net/url"
 	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
+	log "github.com/go-pkgz/lgr"
 	"golang.org/x/crypto/acme/autocert"
 
 	R "github.com/go-pkgz/rest"
@@ -43,7 +43,7 @@ type SSLConfig struct {
 func (s *Rest) httpToHTTPSRouter() chi.Router {
 	log.Printf("[DEBUG] create https-to-http redirect routes")
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer)
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
 	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
 
 	router.Handle("/*", s.redirectHandler())
@@ -57,7 +57,7 @@ func (s *Rest) httpToHTTPSRouter() chi.Router {
 func (s *Rest) httpChallengeRouter(m *autocert.Manager) chi.Router {
 	log.Printf("[DEBUG] create http-challenge routes")
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer)
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
 	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
 
 	router.Handle("/*", m.HTTPHandler(s.redirectHandler()))

backend/app/rest/httperrors.go

@@ -2,23 +2,47 @@ package rest
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"runtime"
 	"strings"
 
 	"github.com/go-chi/render"
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest"
+)
+
+// All error codes for UI mapping and translation
+const (
+	ErrInternal           = 0  // any internal error
+	ErrCommentNotFound    = 1  // can't find comment
+	ErrDecode             = 2  // failed to unmarshal incoming request
+	ErrNoAccess           = 3  // rejected by auth
+	ErrCommentValidation  = 4  // validation failed
+	ErrPostNotFound       = 5  // can't find post
+	ErrSiteNotFound       = 6  // can't find site
+	ErrUserBlocked        = 7  // user blocked
+	ErrReadOnly           = 8  // write failed on read only
+	ErrCommentRejected    = 9  // general error on rejected comment change
+	ErrCommentEditExpired = 10 // too late for edit
+	ErrCommentEditChanged = 11 // parent comment cannot be changed
+	ErrVoteRejected       = 12 // general error on vote rejected
+	ErrVoteSelf           = 13 // vote for own comment
+	ErrVoteDbl            = 14 // already voted for the comment
+	ErrVoteMax            = 15 // too many votes for the comment
+	ErrVoteMinScore       = 16 // min score reached for the comment
+	ErrActionRejected     = 17 // general error for rejected actions
+	ErrAssetNotFound      = 18 // requested file not found
 )
 
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
-	render.Status(r, code)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
+func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	render.Status(r, httpStatusCode)
+	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
 
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
+func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details string, errCode int) string {
 	uinfoStr := ""
 	if user, e := GetUserInfo(r); e == nil {
 		uinfoStr = user.Name + "/" + user.ID + " - "
@@ -40,5 +64,6 @@ func errDetailsMsg(r *http.Request, code int, err error, details string) string
 	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
 		remoteIP = remoteIP[:pos]
 	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
+		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -17,7 +17,7 @@ func TestSendErrorJSON(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
+			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456", 123)
 			return
 		}
 		w.WriteHeader(404)
@@ -33,7 +33,7 @@ func TestSendErrorJSON(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
+	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
 func TestErrorDetailsMsg(t *testing.T) {
@@ -41,8 +41,8 @@ func TestErrorDetailsMsg(t *testing.T) {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
 		require.Nil(t, err)
 		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
 	}
 	callerFn()
 }
@@ -53,8 +53,8 @@ func TestErrorDetailsMsgWithUser(t *testing.T) {
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
 		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,18 +1,18 @@
 package proxy
 
 import (
+	"context"
 	"encoding/base64"
 	"io"
-	"log"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-
+	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
+	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
 )
@@ -23,6 +23,7 @@ type Image struct {
 	RemarkURL string
 	RoutePath string
 	Enabled   bool
+	Timeout   time.Duration
 }
 
 // Convert all img src links without https to proxied links
@@ -48,19 +49,31 @@ func (p Image) Routes() chi.Router {
 	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
 		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
 			return
 		}
 
+		timeout := 60 * time.Second // default
+		if p.Timeout > 0 {
+			timeout = p.Timeout
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+
 		client := http.Client{Timeout: 30 * time.Second}
 		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(func() error {
+		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
 			var e error
-			resp, e = client.Get(string(src))
+			req, e := http.NewRequest("GET", string(src), nil)
+			if e != nil {
+				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
+			}
+			resp, e = client.Do(req.WithContext(ctx))
 			return e
 		})
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
 			return
 		}
 		defer func() {

backend/app/rest/proxy/image_test.go

@@ -3,9 +3,12 @@ package proxy
 import (
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -87,6 +90,25 @@ func TestImage_Routes(t *testing.T) {
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
+func TestImage_RoutesTimedOut(t *testing.T) {
+	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+	router := img.Routes()
+
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	t.Log(string(b))
+	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
+}
+
 func TestPicture_Convert(t *testing.T) {
 	img := Image{Enabled: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
@@ -110,7 +132,13 @@ func imgHTTPServer(t *testing.T) *httptest.Server {
 			t.Log("http img request", r.URL)
 			w.Header().Add("Content-Length", "123")
 			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			assert.NoError(t, err)
+			return
+		}
+		if r.URL.Path == "/image/img-slow.png" {
+			time.Sleep(500 * time.Millisecond)
+			w.WriteHeader(500)
 			return
 		}
 		t.Log("http img request - not found", r.URL)

backend/app/rest/tree.go

@@ -111,8 +111,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 }
 
 // filter returns comments for parentID
-func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) (f []store.Comment) {
-
+func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) []store.Comment {
+	f := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			f = append(f, c)
@@ -151,6 +151,18 @@ func (t *Tree) sortNodes(sortType string) {
 			}
 			return t.Nodes[i].Comment.Score < t.Nodes[j].Comment.Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortType, "-") {
+				if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+					return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+				}
+				return t.Nodes[i].Comment.Controversy > t.Nodes[j].Comment.Controversy
+			}
+			if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+				return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+			}
+			return t.Nodes[i].Comment.Controversy < t.Nodes[j].Comment.Controversy
+
 		default:
 			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
 		}

backend/app/rest/tree_test.go

@@ -3,10 +3,10 @@ package rest
 import (
 	"encoding/json"
 	"io/ioutil"
-	"log"
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -96,15 +96,15 @@ func TestTreeSortNodes(t *testing.T) {
 	comments := []store.Comment{
 		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
+		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2, Controversy: 10},
+		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3, Controversy: 5},
 		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
 		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
 		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
 		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
 		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
+		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2, Controversy: 7},
 		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
 		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
@@ -140,6 +140,19 @@ func TestTreeSortNodes(t *testing.T) {
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
+	res = MakeTree(comments, "+controversy", 0)
+	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
+
+	res = MakeTree(comments, "-controversy", 0)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
+
 	res = MakeTree(comments, "undefined", 0)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)

backend/app/store/admin/admin.go

@@ -3,7 +3,8 @@ package admin
 
 import (
 	"errors"
-	"log"
+
+	log "github.com/go-pkgz/lgr"
 )
 
 // Store defines interface returning admins info for given site

backend/app/store/admin/mongo.go

@@ -1,10 +1,9 @@
 package admin
 
 import (
-	"log"
-
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
+	log "github.com/go-pkgz/lgr"
 
 	"github.com/go-pkgz/mongo"
 )

backend/app/store/comment.go

@@ -10,19 +10,21 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID        string          `json:"id" bson:"_id"`
-	ParentID  string          `json:"pid"`
-	Text      string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
-	Locator   Locator         `json:"locator"`
-	Score     int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
-	Timestamp time.Time       `json:"time" bson:"time"`
-	Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	Pin       bool            `json:"pin,omitempty" bson:"pin,omitempty"`
-	Deleted   bool            `json:"delete,omitempty" bson:"delete"`
-	PostTitle string          `json:"title,omitempty" bson:"title"`
+	ID          string          `json:"id" bson:"_id"`
+	ParentID    string          `json:"pid"`
+	Text        string          `json:"text"`
+	Orig        string          `json:"orig,omitempty"`
+	User        User            `json:"user"`
+	Locator     Locator         `json:"locator"`
+	Score       int             `json:"score"`
+	Votes       map[string]bool `json:"votes,omitempty"`
+	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
+	Controversy float64         `json:"controversy,omitempty"`
+	Timestamp   time.Time       `json:"time" bson:"time"`
+	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string          `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post

backend/app/store/engine/bolt_accessor.go

@@ -3,11 +3,11 @@ package engine
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
@@ -74,6 +74,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
 	}
 	return &result, nil
 }
@@ -127,7 +128,7 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
 			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
 		}
 
-		// set info with countfor post url
+		// set info with the count for post url
 		if _, e = b.setInfo(tx, comment); e != nil {
 			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
 		}
@@ -155,7 +156,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 
 		return bucket.ForEach(func(k, v []byte) error {
 			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
+			if e = json.Unmarshal(v, &comment); e != nil {
 				return errors.Wrap(e, "failed to unmarshal")
 			}
 			comments = append(comments, comment)
@@ -163,13 +164,15 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 		})
 	})
 
-	comments = sortComments(comments, sortFld)
+	comments = SortComments(comments, sortFld)
 	return comments, err
 }
 
 // Last returns up to max last comments for given siteID
 func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
 
+	comments = []store.Comment{}
+
 	if max > lastLimit || max == 0 {
 		max = lastLimit
 	}
@@ -193,7 +196,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
 			}
 
 			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
+			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
 				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
 				continue
 			}
@@ -333,11 +336,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.
 
 	// retrieve comments for refs
 	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
 		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
+		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
 			comments = append(comments, c)
 		}
 	}

backend/app/store/engine/bolt_accessor_test.go

@@ -16,8 +16,8 @@ import (
 var testDb = "test-remark.db"
 
 func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -37,8 +37,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
 }
 
 func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	comment := store.Comment{
 		ID:        "id-ro",
@@ -61,8 +61,8 @@ func TestBoltDB_CreateReadOnly(t *testing.T) {
 }
 
 func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -80,8 +80,9 @@ func TestBoltDB_Get(t *testing.T) {
 }
 
 func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
+
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
 	assert.Nil(t, err)
@@ -107,8 +108,8 @@ func TestBoltDB_Put(t *testing.T) {
 }
 
 func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Last("radio-t", 0)
 	assert.Nil(t, err)
@@ -125,8 +126,8 @@ func TestBoltDB_Last(t *testing.T) {
 }
 
 func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -141,8 +142,8 @@ func TestBoltDB_Count(t *testing.T) {
 }
 
 func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	// add one more for https://radio-t.com/2
 	comment := store.Comment{
@@ -181,8 +182,8 @@ func TestBoltDB_List(t *testing.T) {
 }
 
 func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
 
@@ -224,8 +225,8 @@ func TestBoltDB_Info(t *testing.T) {
 }
 
 func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.User("radio-t", "user1", 5, 0)
 	assert.Nil(t, err)
@@ -253,7 +254,11 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 	os.Remove(testDb)
 	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
-	defer os.Remove(testDb)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}()
 
 	c := store.Comment{
 		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -303,8 +308,9 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 }
 
 func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
+
 	count, err := b.UserCount("radio-t", "user1")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, count)
@@ -343,12 +349,12 @@ func TestBoltDB_New(t *testing.T) {
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	os.Remove(testDb)
 
 	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	assert.Nil(t, err)
-	b := boltStore
+	b = boltStore
 
 	comment := store.Comment{
 		ID:        "id-1",
@@ -370,5 +376,9 @@ func prep(t *testing.T) *BoltDB {
 	_, err = b.Create(comment)
 	assert.Nil(t, err)
 
-	return b
+	teardown = func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}
+	return b, teardown
 }

backend/app/store/engine/bolt_admin.go

@@ -2,10 +2,10 @@ package engine
 
 import (
 	"encoding/json"
-	"log"
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
 		}
 
 		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
+		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
 			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
+		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
 			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
 		}
 
 		// delete from "last" bucket
 		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
+		if err = lastBkt.Delete([]byte(commentID)); err != nil {
 			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
@@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
 			return nil
 		}
 
-		until, err := time.Parse(tsNano, string(val))
-		if err != nil {
+		until, e := time.Parse(tsNano, string(val))
+		if e != nil {
 			blocked = false
 			return nil
 		}
@@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(blocksBucketName))
 		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
+			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+			if errParse != nil {
+				return errors.Wrap(errParse, "can't parse block ts")
 			}
 			if time.Now().Before(ts) {
 				// get user name from comment user section
 				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
+				userComments, errUser := b.User(siteID, string(k), 1, 0)
+				if errUser == nil && len(userComments) > 0 {
 					userName = userComments[0].User.Name
 				}
 				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})

backend/app/store/engine/bolt_admin_test.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	"os"
 	"testing"
 	"time"
 
@@ -12,14 +11,19 @@ import (
 )
 
 func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(res), "initially 2 comments")
 
+	count, err := b.Count(loc)
+	require.NoError(t, err)
+	assert.Equal(t, 2, count, "count=2 initially")
+
 	err = b.Delete(loc, res[0].ID, store.SoftDelete)
 	assert.Nil(t, err)
 
@@ -37,6 +41,10 @@ func TestBoltAdmin_Delete(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
 
+	count, err = b.Count(loc)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
 	err = b.Delete(loc, "123456", store.SoftDelete)
 	assert.NotNil(t, err)
 
@@ -50,8 +58,9 @@ func TestBoltAdmin_Delete(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -70,8 +79,9 @@ func TestBoltAdmin_DeleteHard(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -94,8 +104,10 @@ func TestBoltAdmin_DeleteAll(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
+
 	err := b.DeleteUser("radio-t", "user1")
 	require.NoError(t, err)
 
@@ -122,8 +134,9 @@ func TestBoltAdmin_DeleteUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 
@@ -142,8 +155,10 @@ func TestBoltAdmin_BlockUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
+
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
 	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
@@ -152,8 +167,9 @@ func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
@@ -178,8 +194,9 @@ func TestBoltAdmin_BlockList(t *testing.T) {
 }
 
 func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
 
@@ -198,8 +215,9 @@ func TestBoltAdmin_ReadOnly(t *testing.T) {
 }
 
 func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
 

backend/app/store/engine/engine.go

@@ -62,8 +62,8 @@ const (
 	userLimit = 500
 )
 
-// sortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
+// SortComments is for engines can't sort data internally
+func SortComments(comments []store.Comment, sortFld string) []store.Comment {
 	sort.Slice(comments, func(i, j int) bool {
 		switch sortFld {
 		case "+time", "-time", "time", "+active", "-active", "active":
@@ -84,6 +84,18 @@ func sortComments(comments []store.Comment, sortFld string) []store.Comment {
 			}
 			return comments[i].Score < comments[j].Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortFld, "-") {
+				if comments[i].Controversy == comments[j].Controversy {
+					return comments[i].Timestamp.Before(comments[j].Timestamp)
+				}
+				return comments[i].Controversy > comments[j].Controversy
+			}
+			if comments[i].Controversy == comments[j].Controversy {
+				return comments[i].Timestamp.Before(comments[j].Timestamp)
+			}
+			return comments[i].Controversy < comments[j].Controversy
+
 		default:
 			return comments[i].Timestamp.Before(comments[j].Timestamp)
 		}

backend/app/store/engine/engine_test.go

@@ -11,33 +11,45 @@ import (
 
 func TestEngine_sortComments(t *testing.T) {
 	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
+		{ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
+		{ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
+		{ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
+		{ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
 	}
 
-	sortComments(cc, "+time")
+	SortComments(cc, "+time")
 	assert.Equal(t, "1", cc[0].ID)
 	assert.Equal(t, "2", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-time")
+	SortComments(cc, "-time")
 	assert.Equal(t, "4", cc[0].ID)
 	assert.Equal(t, "3", cc[1].ID)
 	assert.Equal(t, "2", cc[2].ID)
 	assert.Equal(t, "1", cc[3].ID)
 
-	sortComments(cc, "score")
+	SortComments(cc, "score")
 	assert.Equal(t, "2", cc[0].ID)
 	assert.Equal(t, "1", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-score")
+	SortComments(cc, "-score")
 	assert.Equal(t, "3", cc[0].ID)
 	assert.Equal(t, "4", cc[1].ID)
 	assert.Equal(t, "1", cc[2].ID)
 	assert.Equal(t, "2", cc[3].ID)
+
+	SortComments(cc, "controversy")
+	assert.Equal(t, "1", cc[0].ID)
+	assert.Equal(t, "4", cc[1].ID)
+	assert.Equal(t, "2", cc[2].ID)
+	assert.Equal(t, "3", cc[3].ID)
+
+	SortComments(cc, "-controversy")
+	assert.Equal(t, "3", cc[0].ID)
+	assert.Equal(t, "2", cc[1].ID)
+	assert.Equal(t, "1", cc[2].ID)
+	assert.Equal(t, "4", cc[3].ID)
 }

backend/app/store/engine/mongo.go

@@ -5,11 +5,10 @@ import (
 
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
+	"github.com/go-pkgz/mongo"
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
-	"github.com/go-pkgz/mongo"
-
 	"github.com/umputun/remark/backend/app/store"
 )
 
@@ -231,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	for _, m := range metas {
-		ids = append(ids, m.ID)
+	for _, meta := range metas {
+		ids = append(ids, meta.ID)
 	}
 	return ids, nil
 }

backend/app/store/image/fs_store.go

@@ -0,0 +1,175 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"io/ioutil"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+	MaxHeight  int
+	MaxWidth   int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	data, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
+	}
+	if len(data) > f.MaxSize {
+		return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
+	}
+
+	// read header first, needs it to check if data is valid png/gif/jpeg
+	if !isValidImage(data[:512]) {
+		return "", errors.Errorf("file %s is not in allowed format", fileName)
+	}
+
+	data, resized := resize(data, f.MaxWidth, f.MaxHeight)
+
+	id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+	if resized { // resized also converted to png
+		id = strings.TrimSuffix(id, filepath.Ext(id)) + ".png"
+		dst = f.location(f.Staging, id)
+	}
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, fileName, len(data))
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", path, age)
+			return os.Remove(path)
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// SizeLimit returns max size of allowed image
+func (f *FileSystem) SizeLimit() int {
+	return f.MaxSize
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}

backend/app/store/image/fs_store_test.go

@@ -0,0 +1,308 @@
+package image
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveWithResize(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 32, 32
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1142, len(data))
+}
+
+func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 400, 300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 10786, len(data))
+}
+
+func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 1400, 1300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".jpg")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 23983, len(data))
+}
+
+func TestFsStore_WrongFormat(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
+	assert.EqualError(t, err, "file file1.png is not in allowed format")
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 2000
+	_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string) (path string) {
+		id, err := svc.Save(file, user, gopherPNG())
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		assert.Equal(t, 1462, len(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = &FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    1500,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/image.go

@@ -0,0 +1,201 @@
+// Package image handles storing, resizing and retrieval of images
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
+package image
+
+//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
+
+import (
+	"bytes"
+	"context"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	"image/png"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/rs/xid"
+	"golang.org/x/image/draw"
+)
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+	SizeLimit() int                                                          // max image size
+}
+
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	idsFn func() (ids []string)
+	TS    time.Time
+}
+
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil || s == nil {
+		return
+	}
+
+	s.once.Do(func() {
+		log.Printf("[DEBUG] image submitter activated")
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
+				}
+			}
+			log.Printf("[INFO] image submitter terminated")
+		}()
+	})
+
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
+}
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
+		}
+	}
+}
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
+// biggest side (width or height) preserving aspect ratio.
+// Returns original data if resizing is not needed or failed.
+// If resized the result will be for png format and ok flag will be true.
+func resize(data []byte, limitW, limitH int) ([]byte, bool) {
+	if data == nil || limitW <= 0 || limitH <= 0 {
+		return data, false
+	}
+
+	src, _, err := image.Decode(bytes.NewBuffer(data))
+	if err != nil {
+		log.Printf("[WARN] can't decode image, %s", err)
+		return data, false
+	}
+
+	bounds := src.Bounds()
+	w, h := bounds.Dx(), bounds.Dy()
+	if w <= limitW && h <= limitH || w <= 0 || h <= 0 {
+		log.Printf("[DEBUG] resizing image is smaller that the limit or has 0 size")
+		return data, false
+	}
+
+	newW, newH := getProportionalSizes(w, h, limitW, limitH)
+	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
+	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+
+	var out bytes.Buffer
+	if err = png.Encode(&out, m); err != nil {
+		log.Printf("[WARN] can't encode resized image to png, %s", err)
+		return data, false
+	}
+	return out.Bytes(), true
+}
+
+// getProportionalSizes returns width and height resized by both dimensions proportionally
+func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
+
+	if srcW <= limitW && srcH <= limitH {
+		return srcW, srcH
+	}
+
+	ratioW := float64(srcW) / float64(limitW)
+	propH := float64(srcH) / ratioW
+
+	ratioH := float64(srcH) / float64(limitH)
+	propW := float64(srcW) / ratioH
+
+	if int(propH) > limitH {
+		return int(propW), limitH
+	}
+
+	return limitW, int(propH)
+}
+
+// check if file f is a valid image format, i.e. gif, png, jpeg or webp
+func isValidImage(b []byte) bool {
+	ct := http.DetectContentType(b)
+	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
+}
+
+// guid makes a globally unique id
+func guid() string {
+	return xid.New().String()
+}

backend/app/store/image/image_mock.go

@@ -0,0 +1,105 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package image
+
+import context "context"
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import time "time"
+
+// MockStore is an autogenerated mock type for the Store type
+type MockStore struct {
+	mock.Mock
+}
+
+// Cleanup provides a mock function with given fields: ctx, ttl
+func (_m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
+	ret := _m.Called(ctx, ttl)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, time.Duration) error); ok {
+		r0 = rf(ctx, ttl)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Commit provides a mock function with given fields: id
+func (_m *MockStore) Commit(id string) error {
+	ret := _m.Called(id)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(id)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Load provides a mock function with given fields: id
+func (_m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	ret := _m.Called(id)
+
+	var r0 io.ReadCloser
+	if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(io.ReadCloser)
+		}
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(string) int64); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string) error); ok {
+		r2 = rf(id)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: fileName, userID, r
+func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) {
+	ret := _m.Called(fileName, userID, r)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok {
+		r0 = rf(fileName, userID, r)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok {
+		r1 = rf(fileName, userID, r)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SizeLimit provides a mock function with given fields:
+func (_m *MockStore) SizeLimit() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}

backend/app/store/image/image_test.go

@@ -0,0 +1,146 @@
+package image
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"io/ioutil"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{ImageAPI: "/blah/"}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
+func TestService_Cleanup(t *testing.T) {
+	store := MockStore{}
+	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
+
+	svc := Service{Store: &store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+	store.AssertNumberOfCalls(t, "Cleanup", 10)
+}
+
+func TestService_Submit(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 0)
+	time.Sleep(time.Millisecond * 150)
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_Close(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 3)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_resize(t *testing.T) {
+
+	// Reader is nil.
+	resized, ok := resize(nil, 100, 100)
+	assert.Nil(t, resized)
+	assert.False(t, ok)
+
+	// Negative limit error.
+	resized, ok = resize([]byte("some picture bin data"), -1, -1)
+	require.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("some picture bin data"))
+	assert.False(t, ok)
+
+	// Decode error.
+	resized, ok = resize([]byte("invalid image content"), 100, 100)
+	assert.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("invalid image content"))
+	assert.False(t, ok)
+
+	cases := []struct {
+		file   string
+		wr, hr int
+	}{
+		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
+		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
+	}
+
+	for _, c := range cases {
+		img, err := ioutil.ReadFile(c.file)
+		require.Nil(t, err, "can't open test file %s", c.file)
+
+		// No need for resize, image dimensions are smaller than resize limit.
+		resized, ok = resize(img, 800, 800)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.Equal(t, resized, img)
+		assert.False(t, ok)
+
+		// Resizing to half of width. Check resized image format PNG.
+		resized, ok = resize(img, 400, 400)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.True(t, ok)
+
+		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
+		assert.Nil(t, err, "file %s", c.file)
+		assert.Equal(t, "png", format, "file %s", c.file)
+		bounds := imgRz.Bounds()
+		assert.Equal(t, c.wr, bounds.Dx(), "file %s", c.file)
+		assert.Equal(t, c.hr, bounds.Dy(), "file %s", c.file)
+	}
+
+}
+
+func TestGetProportionalSizes(t *testing.T) {
+	tbl := []struct {
+		inpW, inpH     int
+		limitW, limitH int
+		resW, resH     int
+	}{
+		{10, 20, 50, 25, 10, 20},
+		{400, 200, 50, 25, 50, 25},
+		{100, 100, 50, 25, 25, 25},
+		{100, 200, 50, 25, 12, 25},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			resW, resH := getProportionalSizes(tt.inpW, tt.inpH, tt.limitW, tt.limitH)
+			assert.Equal(t, tt.resW, resW, "width")
+			assert.Equal(t, tt.resH, resH, "height")
+		})
+	}
+}

backend/app/store/service/restricted_words.go

@@ -0,0 +1,176 @@
+package service
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// RestrictedWordsLister provides restricted words in comments per site
+type RestrictedWordsLister interface {
+	List(siteID string) (restricted []string, err error)
+}
+
+// StaticRestrictedWordsLister provides same restricted words in comments for every site
+type StaticRestrictedWordsLister struct {
+	Words []string
+}
+
+// List provides restricted words in comments (ignores siteID)
+func (l StaticRestrictedWordsLister) List(siteID string) (restricted []string, err error) {
+	return l.Words, nil
+}
+
+// RestrictedWordsMatcher matches comment text against restricted words
+type RestrictedWordsMatcher struct {
+	lister RestrictedWordsLister
+}
+
+// NewRestrictedWordsMatcher creates new RestrictedWordsMatcher using provided RestrictedWordsLister
+func NewRestrictedWordsMatcher(lister RestrictedWordsLister) *RestrictedWordsMatcher {
+	return &RestrictedWordsMatcher{lister: lister}
+}
+
+// Match matches comment text against restricted words for specified site
+func (m *RestrictedWordsMatcher) Match(siteID string, text string) bool {
+	restrictedWords, err := m.lister.List(siteID)
+	if err != nil {
+		log.Printf("[WARN] failed to get restricted patterns for site %s: %v", siteID, err)
+		return false
+	}
+	if len(restrictedWords) == 0 {
+		return false
+	}
+
+	tokens := m.tokenize(text)
+	trie := newWildcardTrie(restrictedWords...)
+
+	for _, token := range tokens {
+		if trie.check(token) {
+			return true
+		}
+	}
+	return false
+}
+
+func (m *RestrictedWordsMatcher) tokenize(text string) []string {
+	tokens := make([]string, 0, 10) // accumulator for tokens
+	word := false                   // flag shows if current range is word
+	start := 0                      // beginning of the current range
+
+	for pos, r := range text {
+		if unicode.IsLetter(r) || unicode.IsNumber(r) {
+			if !word {
+				// everything from start to pos - 1 is not a word, so reset start and start word tracking
+				start = pos
+				word = true
+			}
+			continue
+		}
+
+		if word && start < pos {
+			// everything from start to pos - 1 is a word, so add it as a token and reset start
+			tokens = append(tokens, strings.ToLower(text[start:pos]))
+			start = pos
+		}
+
+		// exited the word
+		word = false
+	}
+
+	// since we append tokens when we already left the word (on next iteration),
+	// we need to do it manually for the last iteration
+	if word {
+		tokens = append(tokens, strings.ToLower(text[start:]))
+	}
+
+	return tokens
+}
+
+type wildcardTrie struct {
+	terminal bool
+	children map[rune]*wildcardTrie
+}
+
+func newWildcardTrie(patterns ...string) *wildcardTrie {
+	trie := &wildcardTrie{terminal: false, children: make(map[rune]*wildcardTrie)}
+	for _, p := range patterns {
+		trie.addPattern(p)
+	}
+	return trie
+}
+
+func (trie *wildcardTrie) addPattern(pattern string) {
+	// since pattern matching algorithm is recursive we do not allow long patterns
+	if utf8.RuneCountInString(pattern) < 1 || utf8.RuneCountInString(pattern) > 64 {
+		log.Printf("[WARN] invalid pattern length '%s': actual - %d, min allowed - 1, max allowed - 64", pattern, utf8.RuneCountInString(pattern))
+		return
+	}
+
+	node := trie
+
+	for _, r := range strings.ToLower(strings.TrimSpace(pattern)) {
+		if childNode, exists := node.children[r]; exists {
+			node = childNode
+			continue
+		}
+
+		childNode := newWildcardTrie()
+		node.children[r] = childNode
+		node = childNode
+	}
+
+	node.terminal = true
+}
+
+// check tests if any pattern stored in trie matches the token. Recursive. Max depth is longest pattern in trie.
+func (trie *wildcardTrie) check(token string) bool {
+	if len(token) == 0 {
+		if trie.terminal {
+			return true
+		}
+
+		if childNode, exists := trie.children['*']; exists && childNode.terminal {
+			return true
+		}
+
+		return false
+	}
+
+	r, width := utf8.DecodeRuneInString(token)
+
+	if childNode, exists := trie.children[r]; exists {
+		if childNode.check(token[width:]) {
+			return true
+		}
+	}
+
+	if childNode, exists := trie.children['*']; exists {
+		if childNode.terminal {
+			return true
+		}
+		if childNode.checkAllSuffixes(token) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (trie *wildcardTrie) checkAllSuffixes(token string) bool {
+	suffix := token
+	for {
+		if len(suffix) == 0 {
+			return false
+		}
+
+		if trie.check(suffix) {
+			return true
+		}
+
+		_, width := utf8.DecodeRuneInString(suffix)
+		suffix = suffix[width:]
+	}
+}

backend/app/store/service/restricted_words_test.go

@@ -0,0 +1,77 @@
+package service
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMatcher_Tokenize(t *testing.T) {
+
+	matcher := NewRestrictedWordsMatcher(StaticRestrictedWordsLister{})
+
+	tbl := []struct {
+		input  string
+		output []string
+	}{
+		{
+			"   word0 word1 word2, word3,,, !word4 !word5? word6-word7 word8",
+			[]string{"word0", "word1", "word2", "word3", "word4", "word5", "word6", "word7", "word8"},
+		},
+		{"русский 中文 française ไทย", []string{"русский", "中文", "française", "ไทย"}},
+		{"word", []string{"word"}},
+		{"", []string{}},
+		{"\t\t\n\t \n\t \r\n    \t ,,, !#$%^&*()", []string{}},
+		{"👍", []string{}},
+	}
+
+	for _, td := range tbl {
+		tokens := matcher.tokenize(td.input)
+		assert.Equal(t, td.output, tokens, "unexpected result for input '%v'", td.input)
+	}
+}
+
+func TestWildcardTrie_Check(t *testing.T) {
+
+	tbl := []struct {
+		input   []string
+		match   []string
+		nomatch []string
+	}{
+		{[]string{"abc", "abb", "aab"}, []string{"abc", "abb", "aab"}, []string{"aaa", "aaaa", "a", "ab"}},
+		{[]string{"abc", "*ck", "*z"}, []string{"abc", "duck", "quack", "ck", "xyz"}, []string{"quacker", "buzzer"}},
+		{[]string{"abc", "du*", "c*"}, []string{"abc", "duck", "dungeon", "du", "cup"}, []string{"bbc", "ddu", "scuba"}},
+		{[]string{"abc", "*uc*", "*x*"}, []string{"abc", "duck", "stuck", "uc", "wwxww", "xww", "wwx"}, []string{"bbc", "duke"}},
+		{[]string{"abc", "d*k", "st*ck"}, []string{"abc", "duck", "dk", "stck", "stuck", "stiiick"}, []string{"bbc", "adka", "st", "ck"}},
+		{[]string{"abc", "*a*a*"}, []string{"abc", "safari", "banana", "aa"}, []string{"bbc", "car", "a"}},
+		{
+			[]string{"ложить", "при*", "*ий", "*бег*", "про*жа", "*ไ*ย*", "*請*请*"},
+			[]string{"ложить", "приклад", "ихний", "прибегать", "пропажа", "ไทย", "ทไย", "ไยท", "請問请问"},
+			[]string{"положить", "гранпри", "бийск", "請", "ยไท"},
+		},
+	}
+
+	for _, td := range tbl {
+		n := newWildcardTrie(td.input...)
+
+		for _, token := range td.match {
+			assert.True(t, n.check(token), "should match token '%s' for restricted words '%v'", token, td.input)
+		}
+
+		for _, token := range td.nomatch {
+			assert.False(t, n.check(token), "should not match token '%s' for restricted words '%v'", token, td.input)
+		}
+	}
+}
+
+func TestMatcher_MatchIfContainsRestrictedWords(t *testing.T) {
+	matcher := NewRestrictedWordsMatcher(StaticRestrictedWordsLister{[]string{"duck"}})
+	text := "What the duck it that?"
+	assert.True(t, matcher.Match("fakeID", text))
+}
+
+func TestMatcher_DoNotMatchIfNoRestrictedWords(t *testing.T) {
+	matcher := NewRestrictedWordsMatcher(StaticRestrictedWordsLister{[]string{"quack"}})
+	text := "What the duck it that?"
+	assert.False(t, matcher.Match("fakeID", text))
+}

backend/app/store/service/service.go

@@ -1,14 +1,18 @@
 package service
 
 import (
-	"log"
+	"math"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	multierror "github.com/hashicorp/go-multierror"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -18,11 +22,14 @@ import (
 // DataStore wraps store.Interface with additional methods
 type DataStore struct {
 	engine.Interface
-	EditDuration   time.Duration
-	AdminStore     admin.Store
-	MaxCommentSize int
-	MaxVotes       int
-	TitleExtractor *TitleExtractor
+	EditDuration           time.Duration
+	AdminStore             admin.Store
+	MaxCommentSize         int
+	MaxVotes               int
+	PositiveScore          bool
+	TitleExtractor         *TitleExtractor
+	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -30,6 +37,11 @@ type DataStore struct {
 		sync.Once
 		locks map[string]sync.Locker
 	}
+
+	repliesCache struct {
+		*cache.Cache
+		once sync.Once
+	}
 }
 
 // UserMetaData keeps info about user flags
@@ -49,10 +61,14 @@ type PostMetaData struct {
 }
 
 const defaultCommentMaxSize = 2000
+const maxLastCommentsReply = 1000
 
 // UnlimitedVotes doesn't restrict MaxVotes
 const UnlimitedVotes = -1
 
+// ErrRestrictedWordsFound returned in case comment text contains restricted words
+var ErrRestrictedWordsFound = errors.New("comment contains restricted words")
+
 // Create prepares comment and forward to Interface.Create
 func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
 
@@ -60,18 +76,48 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", errors.Wrap(err, "failed to prepare comment")
 	}
 
-	// keep input title and set to extracted if missing
-	if s.TitleExtractor != nil && comment.PostTitle == "" {
-		if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
-			comment.PostTitle = title
-		} else {
-			log.Printf("[WARN] failed to set title, %v", err)
-		}
+	if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, comment.Text) {
+		return "", ErrRestrictedWordsFound
 	}
 
+	func() { // keep input title and set to extracted if missing
+		if s.TitleExtractor == nil || comment.PostTitle != "" {
+			return
+		}
+		title, e := s.TitleExtractor.Get(comment.Locator.URL)
+		if e != nil {
+			log.Printf("[WARN] failed to set title, %v", e)
+			return
+		}
+		comment.PostTitle = title
+	}()
+
+	s.submitImages(comment)
 	return s.Interface.Create(comment)
 }
 
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			return nil
+		}
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+}
+
 // prepareNewComment sets new comment fields, hashing and sanitizing data
 func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) {
 	// fill ID and time if empty
@@ -108,8 +154,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool)
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
 	comment, err = s.Get(locator, commentID)
@@ -139,6 +185,10 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		return comment, errors.Errorf("maximum number of votes exceeded for comment %s", commentID)
 	}
 
+	if s.PositiveScore && comment.Score <= 0 && !val {
+		return comment, errors.Errorf("minimal score reached for comment %s", commentID)
+	}
+
 	// reset vote if user changed to opposite
 	if voted && v != val {
 		delete(comment.Votes, userID)
@@ -156,9 +206,36 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		comment.Score--
 	}
 
+	comment.Vote = 0
+	if v, ok := comment.Votes[userID]; ok {
+		if v {
+			comment.Vote = 1
+		} else {
+			comment.Vote = -1
+		}
+	}
+
+	comment.Controversy = s.controversy(s.upsAndDowns(comment))
+
 	return comment, s.Put(locator, comment)
 }
 
+// controversy calculates controversial index of votes
+// source - https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L60
+func (s *DataStore) controversy(ups, downs int) float64 {
+
+	if downs <= 0 || ups <= 0 {
+		return 0
+	}
+
+	magnitude := ups + downs
+	balance := float64(downs) / float64(ups)
+	if ups <= downs {
+		balance = float64(ups) / float64(downs)
+	}
+	return math.Pow(float64(magnitude), balance)
+}
+
 // EditRequest contains fields needed for comment update
 type EditRequest struct {
 	Text    string
@@ -179,11 +256,19 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 		return comment, errors.Errorf("too late to edit %s", commentID)
 	}
 
+	if s.HasReplies(comment) {
+		return comment, errors.Errorf("parent comment with reply can't be edited, %s", commentID)
+	}
+
 	if req.Delete { // delete request
 		comment.Deleted = true
 		return comment, s.Delete(locator, commentID, store.SoftDelete)
 	}
 
+	if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) {
+		return comment, ErrRestrictedWordsFound
+	}
+
 	comment.Text = req.Text
 	comment.Orig = req.Orig
 	comment.Edit = &store.Edit{
@@ -196,6 +281,37 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 	return comment, err
 }
 
+// HasReplies checks if there is any reply to the comments
+// Loads last maxLastCommentsReply comments and compare parent id to the comment's id
+// Comments with replies cached for 5 minutes
+func (s *DataStore) HasReplies(comment store.Comment) bool {
+
+	s.repliesCache.once.Do(func() {
+		//  default expiration time of 5 minutes, purge every 10 minutes
+		s.repliesCache.Cache = cache.New(5*time.Minute, 10*time.Minute)
+	})
+
+	if _, found := s.repliesCache.Get(comment.ID); found {
+		return true
+	}
+
+	comments, err := s.Last(comment.Locator.SiteID, maxLastCommentsReply)
+	if err != nil {
+		log.Printf("[WARN] can't get last comments for reply check, %v", err)
+		return false
+	}
+
+	for _, c := range comments {
+		if c.ParentID != "" && !c.Deleted {
+			if c.ParentID == comment.ID {
+				s.repliesCache.Set(comment.ID, true, cache.DefaultExpiration)
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // SetTitle puts title from the locator.URL page and overwrites any existing title
 func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error) {
 	if s.TitleExtractor == nil {
@@ -336,8 +452,45 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
 	return errs.ErrorOrNil()
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+// Find wraps engine's Find call and alter results if needed
+func (s *DataStore) Find(locator store.Locator, sort string) ([]store.Comment, error) {
+	comments, err := s.Interface.Find(locator, sort)
+	if err != nil {
+		return comments, err
+	}
+
+	changedSort := false
+	// set votes controversy for comments added prior to #274
+	for i, c := range comments {
+		if c.Controversy == 0 && len(c.Votes) > 0 {
+			comments[i].Controversy = s.controversy(s.upsAndDowns(c))
+			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+				changedSort = true
+			}
+		}
+	}
+
+	// resort commits if altered
+	if changedSort {
+		comments = engine.SortComments(comments, sort)
+	}
+
+	return comments, nil
+}
+
+func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
+	for _, v := range c.Votes {
+		if v {
+			ups++
+			continue
+		}
+		downs++
+	}
+	return ups, downs
+}
+
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()

backend/app/store/service/service_test.go

@@ -12,20 +12,23 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 var testDb = "/tmp/test-remark.db"
 
 func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -45,11 +48,11 @@ func TestService_CreateFromEmpty(t *testing.T) {
 	assert.Equal(t, "user", res.User.ID)
 	assert.Equal(t, "name", res.User.Name)
 	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, map[string]bool{}, res.Votes)
+	assert.Equal(t, map[string]bool(nil), res.Votes)
 }
 
 func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -76,7 +79,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
@@ -106,7 +109,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 }
 
 func TestService_SetTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -114,11 +117,13 @@ func TestService_SetTitle(t *testing.T) {
 			w.WriteHeader(404)
 		}
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -145,6 +150,8 @@ func TestService_SetTitle(t *testing.T) {
 	t.Logf("%+v", res)
 	assert.Equal(t, "", res.PostTitle)
 
+	b.TitleExtractor.cache.Purge()
+
 	atomic.StoreInt32(&titleEnable, 1)
 	c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
 	require.NoError(t, err)
@@ -156,7 +163,7 @@ func TestService_SetTitle(t *testing.T) {
 }
 
 func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -172,11 +179,13 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, c.Score)
+	assert.Equal(t, 1, c.Vote)
 	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
 
 	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
@@ -190,6 +199,8 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 1, res[0].Score)
+	assert.Equal(t, 1, res[0].Vote)
+	assert.Equal(t, 0.0, res[0].Controversy)
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
 	assert.Nil(t, err, "vote reset")
@@ -197,11 +208,12 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "vote reset ok")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "vote reset ok")
 }
 
 func TestService_VoteLimit(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -219,7 +231,7 @@ func TestService_VoteLimit(t *testing.T) {
 }
 
 func TestService_VotesDisabled(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -227,7 +239,7 @@ func TestService_VotesDisabled(t *testing.T) {
 }
 
 func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -243,7 +255,7 @@ func TestService_VoteAggressive(t *testing.T) {
 	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	// add a vote as user2
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
@@ -255,7 +267,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 		}()
 	}
 	wg.Wait()
@@ -274,7 +286,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
 		}()
 	}
 	wg.Wait()
@@ -287,7 +299,7 @@ func TestService_VoteAggressive(t *testing.T) {
 
 func TestService_VoteConcurrent(t *testing.T) {
 
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -304,21 +316,92 @@ func TestService_VoteConcurrent(t *testing.T) {
 	var wg sync.WaitGroup
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		i := i
+		ii := i
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
+				fmt.Sprintf("user1-%d", ii), true)
 		}()
 	}
 	wg.Wait()
 	res, err = b.Last("radio-t", 0)
 	require.NoError(t, err)
-	assert.Equal(t, 100, res[0].Score, "should have 1000 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 1000 votes")
+	assert.Equal(t, 100, res[0].Score, "should have 100 score")
+	assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
+	assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
+}
+
+func TestService_VotePositive(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: true}
+
+	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.EqualError(t, err, "minimal score reached for comment id-1")
+
+	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
+	assert.Nil(t, err, "minimal score doesn't affect positive vote")
+
+	b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: false}
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.Nil(t, err, "minimal score ignored")
+	assert.Equal(t, -1, c.Score)
+	assert.Equal(t, 0.0, c.Controversy)
+}
+
+func TestService_VoteControversy(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
+	assert.NoError(t, err)
+	assert.Equal(t, -1, c.Score, "should have -1 score")
+	assert.InDelta(t, 0.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user3", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c.Score, "should have 0 score")
+	assert.InDelta(t, 2.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c.Score, "should have 1 score")
+	assert.InDelta(t, 1.73, c.Controversy, 0.01)
+
+	// check if stored
+	res, err := b.Last("radio-t", 0)
+	require.NoError(t, err)
+	assert.Equal(t, 1, res[0].Score, "should have 1 score")
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+}
+
+func TestService_Controversy(t *testing.T) {
+	tbl := []struct {
+		ups, downs int
+		res        float64
+	}{
+		{0, 0, 0},
+		{10, 5, 3.87},
+		{20, 5, 2.24},
+		{20, 50, 5.47},
+		{20, 0, 0},
+		{1100, 500, 28.60},
+		{1100, 12100, 2.37},
+		{100, 100, 200},
+		{101, 101, 202},
+	}
+
+	b := DataStore{}
+	for i, tt := range tbl {
+		t.Run(fmt.Sprintf("check-%d-%d:%d", i, tt.ups, tt.downs), func(t *testing.T) {
+			assert.InDelta(t, tt.res, b.controversy(tt.ups, tt.downs), 0.01)
+		})
+	}
 }
 
 func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -342,7 +425,7 @@ func TestService_Pin(t *testing.T) {
 }
 
 func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -369,7 +452,7 @@ func TestService_EditComment(t *testing.T) {
 }
 
 func TestService_DeleteComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -388,7 +471,7 @@ func TestService_DeleteComment(t *testing.T) {
 }
 
 func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -404,6 +487,32 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
 	assert.NotNil(t, err)
 }
 
+func TestService_EditCommentReplyFailed(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+
+	res, err := b.Last("radio-t", 0)
+	t.Logf("%+v", res[1])
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Nil(t, res[1].Edit)
+
+	reply := store.Comment{
+		ID:        "123456",
+		ParentID:  "id-1",
+		Text:      "some text",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+	_, err = b.Create(reply)
+	assert.NoError(t, err)
+
+	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID,
+		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
+	assert.EqualError(t, err, "parent comment with reply can't be edited, id-1")
+}
+
 func TestService_ValidateComment(t *testing.T) {
 
 	b := DataStore{MaxCommentSize: 2000, AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -430,7 +539,7 @@ func TestService_ValidateComment(t *testing.T) {
 }
 
 func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := prepStoreEngine(t) // two comments for https://radio-t.com
 
 	// add one more for https://radio-t.com/2
@@ -459,7 +568,7 @@ func TestService_Counts(t *testing.T) {
 }
 
 func TestService_GetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -490,7 +599,7 @@ func TestService_GetMetas(t *testing.T) {
 }
 
 func TestService_SetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -514,7 +623,7 @@ func TestService_SetMetas(t *testing.T) {
 }
 
 func TestService_IsAdmin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
@@ -523,9 +632,99 @@ func TestService_IsAdmin(t *testing.T) {
 	assert.True(t, b.IsAdmin("radio-t", "user2"))
 }
 
+func TestService_HasReplies(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	assert.False(t, b.HasReplies(comment))
+
+	reply := store.Comment{
+		ID:        "123456",
+		ParentID:  "id-1",
+		Text:      "some text",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name 2"},
+	}
+	_, err := b.Create(reply)
+	assert.NoError(t, err)
+	assert.True(t, b.HasReplies(comment))
+}
+
+func TestService_Find(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "123456",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+		Score:     1,
+		Votes:     map[string]bool{"id-1": true, "id-2": true, "123456": false},
+	}
+	_, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy
+	assert.Nil(t, err)
+
+	// make sure Controversy altered
+	res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy")
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, "123456", res[0].ID)
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+	assert.Equal(t, "id-1", res[1].ID)
+	assert.InDelta(t, 0, res[1].Controversy, 0.01)
+}
+
+func TestService_submitImages(t *testing.T) {
+	defer teardown(t)
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	mockStore := image.MockStore{}
+	mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil)
+	imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
+
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+
+	b.submitImages(c)
+	time.Sleep(250 * time.Millisecond)
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
+	_ = os.Remove(testDb)
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -553,3 +752,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 
 	return b
 }
+
+func teardown(_ *testing.T) {
+	_ = os.Remove(testDb)
+}

backend/app/store/service/title.go

@@ -2,20 +2,24 @@ package service
 
 import (
 	"io"
-	"log"
 	"net/http"
+	"time"
 
-	"github.com/go-pkgz/rest/cache"
+	"github.com/go-pkgz/lcw"
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"golang.org/x/net/html"
 )
 
-const teMaxCachedRecs = 1000
+const (
+	teCacheMaxRecs = 1000
+	teCacheTTL     = 15 * time.Minute
+)
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
 	client http.Client
-	cache  cache.LoadingCache
+	cache  lcw.LoadingCache
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
@@ -24,19 +28,19 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {
 		client: client,
 	}
 	var err error
-	res.cache, err = cache.NewMemoryCache(cache.MaxKeys(teMaxCachedRecs))
+	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
-		log.Printf("[WARN] failed to make cache, %v", err)
-		res.cache = &cache.Nop{}
+		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
+		res.cache = &lcw.Nop{}
 	}
 	return &res
 }
 
 // Get page for url and return title
 func (t *TitleExtractor) Get(url string) (string, error) {
-
-	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
-		resp, err := t.client.Get(url)
+	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
+	b, err := t.cache.Get(url, func() (lcw.Value, error) {
+		resp, err := client.Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
@@ -49,14 +53,16 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if !ok {
 			return nil, errors.Errorf("can't get title for %s", url)
 		}
-		return []byte(title), nil
+		return title, nil
 	})
 
+	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
+		_, _ = t.cache.Get(url, func() (lcw.Value, error) { return "", nil })
 		return "", err
 	}
 
-	return string(b), nil
+	return b.(string), nil
 }
 
 // get title from body reader, traverse recursively

backend/app/store/service/title_test.go

@@ -1,14 +1,17 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/syncs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +44,8 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -55,9 +59,59 @@ func TestTitle_Get(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/good")
-		require.Nil(t, err)
-		assert.Equal(t, "blah 123", title)
+		r, e := ex.Get(ts.URL + "/good")
+		require.Nil(t, e)
+		assert.Equal(t, "blah 123", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
+
+func TestTitle_GetConcurrent(t *testing.T) {
+	body := ""
+	for n := 0; n < 1000; n++ {
+		body += "something something blah blah\n"
+	}
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.String(), "/good") {
+			atomic.AddInt32(&hits, 1)
+			_, err := w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			assert.NoError(t, err)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	g := syncs.NewSizedGroup(10)
+
+	for i := 0; i < 100; i++ {
+		ii := i
+		g.Go(func(_ context.Context) {
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(ii))
+			require.Nil(t, err)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(ii), title)
+		})
+	}
+	g.Wait()
+	assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
+}
+
+func TestTitle_GetFailed(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(404)
+	}))
+
+	_, err := ex.Get(ts.URL + "/bad")
+	require.NotNil(t, err)
+
+	for i := 0; i < 100; i++ {
+		r, e := ex.Get(ts.URL + "/bad")
+		require.Nil(t, e)
+		assert.Equal(t, "", r)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
+}

backend/app/store/user.go

@@ -2,14 +2,15 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1"
+	"crypto/sha1" //nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
 	"hash/crc64"
 	"io"
-	"log"
 	"regexp"
+
+	log "github.com/go-pkgz/lgr"
 )
 
 // User holds user-related info
@@ -40,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id)
+	return hashWithFallback(sha1.New(), id) //nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -0,0 +1,41 @@
+module github.com/umputun/remark/backend
+
+go 1.12
+
+require (
+	github.com/PuerkitoBio/goquery v1.4.0
+	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/coreos/bbolt v1.3.0
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/didip/tollbooth v4.0.0+incompatible
+	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
+	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
+	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/cors v1.0.0
+	github.com/go-chi/render v1.0.0
+	github.com/go-pkgz/auth v0.5.0
+	github.com/go-pkgz/lcw v0.2.0
+	github.com/go-pkgz/lgr v0.6.2
+	github.com/go-pkgz/mongo v1.1.2
+	github.com/go-pkgz/repeater v1.1.1
+	github.com/go-pkgz/rest v1.4.0
+	github.com/go-pkgz/syncs v1.1.0
+	github.com/golang/mock v1.2.0
+	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
+	github.com/gorilla/feeds v1.1.0
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
+	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
+	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.8.1
+	github.com/rakyll/statik v0.1.3
+	github.com/rs/xid v1.2.1
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b
+	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	gopkg.in/russross/blackfriday.v2 v2.0.0
+)

backend/go.sum

@@ -0,0 +1,117 @@
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
+github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
+github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
+github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
+github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.5.0 h1:+wqppq35x83PchZNZ7SHHYLI/e8WeETFouujDLsklac=
+github.com/go-pkgz/auth v0.5.0/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
+github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
+github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
+github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
+github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
+github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
+github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
+github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
+github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
+github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
+github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
+golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
+gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=

backend/remark.rest

@@ -1,9 +1,13 @@
 
 ### find request with tree
-GET {{host}}/api/v1/find?site={{site}}&sort=-active&format=tree&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=tree&url={{url}}
 
 ### find request with plain
-GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=plain&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+
+### find request with plain
+GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### last 50 comments
 GET {{host}}/api/v1/last/50?site={{site}}
@@ -41,7 +45,7 @@ Content-Type: application/json
 }
 
 ### update comment
-PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site=remark&url=https://radio-t.com/blah1
+PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site={{site}}&url=https://radio-t.com/blah1
 Content-Type: application/json
 
 {
@@ -50,22 +54,23 @@ Content-Type: application/json
 }
 
 ### pin comment
-PUT {{host}}/api/v1/admin/pin/3665976683?site=remark&url={{url}}&pin=1
+PUT {{host}}/api/v1/admin/pin/3665976683?site={{site}}&url={{url}}&pin=1
 
 ### vote for comment
-PUT {{host}}/api/v1/vote/3665976683?site=remark&url={{url}}&vote=1
+PUT http://127.0.0.1:8080/api/v1/vote/8a8c0b80-0d0a-41c3-84ad-f4034704e827?site={{site}}&url={{url}}&vote=-1
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### get user info
 GET {{host}}/api/v1/user
 
 ### get comment by id
-GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
+GET {{host}}/api/v1/id/73e346f4-d57d-41a8-8803-6671aa187d8e?site={{site}}&url={{url}}
 
 ### get comment by id 2
-GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
+GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site={{site}}&url=https://radio-t.com/blah1
 
 ### get comment by user id
-GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
+GET {{host}}/api/v1/comments?site={{site}}&user={{user}}&limit=5
 
 ### get comment by user id2
 GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
@@ -84,7 +89,7 @@ Content-Type: application/json
 ]
 
 ### list commented posts
-GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
+GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=0
 
 ### block user
 PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -22,6 +22,7 @@ David Symonds <dsymonds@golang.org>
 Filippo Valsorda <hi@filippo.io>
 Glenn Lewis <gmlewis@google.com>
 Ingo Oeser <nightlyone@googlemail.com>
+James Hall <james.hall@shopify.com>
 Johan Euphrosine <proppy@google.com>
 Jonathan Amsterdam <jba@google.com>
 Kunpei Sakai <namusyaka@gmail.com>

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -1,4 +1,4 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
+// Copyright 2014 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
 package metadata // import "cloud.google.com/go/compute/metadata"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -31,9 +32,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"golang.org/x/net/context"
-	"golang.org/x/net/context/ctxhttp"
 )
 
 const (
@@ -64,7 +62,7 @@ var (
 )
 
 var (
-	metaClient = &http.Client{
+	defaultClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
@@ -72,15 +70,15 @@ var (
 			}).Dial,
 			ResponseHeaderTimeout: 2 * time.Second,
 		},
-	}
-	subscribeClient = &http.Client{
+	}}
+	subscribeClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).Dial,
 		},
-	}
+	}}
 )
 
 // NotDefinedError is returned when requested metadata is not defined.
@@ -95,74 +93,16 @@ func (suffix NotDefinedError) Error() string {
 	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
 }
 
-// Get returns a value from the metadata service.
-// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
-//
-// If the GCE_METADATA_HOST environment variable is not defined, a default of
-// 169.254.169.254 will be used instead.
-//
-// If the requested metadata is not defined, the returned error will
-// be of type NotDefinedError.
-func Get(suffix string) (string, error) {
-	val, _, err := getETag(metaClient, suffix)
-	return val, err
-}
-
-// getETag returns a value from the metadata service as well as the associated
-// ETag using the provided client. This func is otherwise equivalent to Get.
-func getETag(client *http.Client, suffix string) (value, etag string, err error) {
-	// Using a fixed IP makes it very difficult to spoof the metadata service in
-	// a container, which is an important use-case for local testing of cloud
-	// deployments. To enable spoofing of the metadata service, the environment
-	// variable GCE_METADATA_HOST is first inspected to decide where metadata
-	// requests shall go.
-	host := os.Getenv(metadataHostEnv)
-	if host == "" {
-		// Using 169.254.169.254 instead of "metadata" here because Go
-		// binaries built with the "netgo" tag and without cgo won't
-		// know the search suffix for "metadata" is
-		// ".google.internal", and this IP address is documented as
-		// being stable anyway.
-		host = metadataIP
-	}
-	url := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("Metadata-Flavor", "Google")
-	req.Header.Set("User-Agent", userAgent)
-	res, err := client.Do(req)
-	if err != nil {
-		return "", "", err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == http.StatusNotFound {
-		return "", "", NotDefinedError(suffix)
-	}
-	if res.StatusCode != 200 {
-		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
-	all, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", "", err
-	}
-	return string(all), res.Header.Get("Etag"), nil
-}
-
-func getTrimmed(suffix string) (s string, err error) {
-	s, err = Get(suffix)
-	s = strings.TrimSpace(s)
-	return
-}
-
-func (c *cachedValue) get() (v string, err error) {
+func (c *cachedValue) get(cl *Client) (v string, err error) {
 	defer c.mu.Unlock()
 	c.mu.Lock()
 	if c.v != "" {
 		return c.v, nil
 	}
 	if c.trim {
-		v, err = getTrimmed(c.k)
+		v, err = cl.getTrimmed(c.k)
 	} else {
-		v, err = Get(c.k)
+		v, err = cl.Get(c.k)
 	}
 	if err == nil {
 		c.v = v
@@ -201,7 +141,7 @@ func testOnGCE() bool {
 	go func() {
 		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
 		req.Header.Set("User-Agent", userAgent)
-		res, err := ctxhttp.Do(ctx, metaClient, req)
+		res, err := defaultClient.hc.Do(req.WithContext(ctx))
 		if err != nil {
 			resc <- false
 			return
@@ -266,6 +206,255 @@ func systemInfoSuggestsGCE() bool {
 	return name == "Google" || name == "Google Compute Engine"
 }
 
+// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no
+// ResponseHeaderTimeout).
+func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	return subscribeClient.Subscribe(suffix, fn)
+}
+
+// Get calls Client.Get on the default client.
+func Get(suffix string) (string, error) { return defaultClient.Get(suffix) }
+
+// ProjectID returns the current instance's project ID string.
+func ProjectID() (string, error) { return defaultClient.ProjectID() }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() }
+
+// InternalIP returns the instance's primary internal IP address.
+func InternalIP() (string, error) { return defaultClient.InternalIP() }
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func Hostname() (string, error) { return defaultClient.Hostname() }
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() }
+
+// InstanceID returns the current VM's numeric instance ID.
+func InstanceID() (string, error) { return defaultClient.InstanceID() }
+
+// InstanceName returns the current VM's instance ID string.
+func InstanceName() (string, error) { return defaultClient.InstanceName() }
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func Zone() (string, error) { return defaultClient.Zone() }
+
+// InstanceAttributes calls Client.InstanceAttributes on the default client.
+func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() }
+
+// ProjectAttributes calls Client.ProjectAttributes on the default client.
+func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() }
+
+// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client.
+func InstanceAttributeValue(attr string) (string, error) {
+	return defaultClient.InstanceAttributeValue(attr)
+}
+
+// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client.
+func ProjectAttributeValue(attr string) (string, error) {
+	return defaultClient.ProjectAttributeValue(attr)
+}
+
+// Scopes calls Client.Scopes on the default client.
+func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) }
+
+func strsContains(ss []string, s string) bool {
+	for _, v := range ss {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// A Client provides metadata.
+type Client struct {
+	hc *http.Client
+}
+
+// NewClient returns a Client that can be used to fetch metadata. All HTTP requests
+// will use the given http.Client instead of the default client.
+func NewClient(c *http.Client) *Client {
+	return &Client{hc: c}
+}
+
+// getETag returns a value from the metadata service as well as the associated ETag.
+// This func is otherwise equivalent to Get.
+func (c *Client) getETag(suffix string) (value, etag string, err error) {
+	// Using a fixed IP makes it very difficult to spoof the metadata service in
+	// a container, which is an important use-case for local testing of cloud
+	// deployments. To enable spoofing of the metadata service, the environment
+	// variable GCE_METADATA_HOST is first inspected to decide where metadata
+	// requests shall go.
+	host := os.Getenv(metadataHostEnv)
+	if host == "" {
+		// Using 169.254.169.254 instead of "metadata" here because Go
+		// binaries built with the "netgo" tag and without cgo won't
+		// know the search suffix for "metadata" is
+		// ".google.internal", and this IP address is documented as
+		// being stable anyway.
+		host = metadataIP
+	}
+	url := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Metadata-Flavor", "Google")
+	req.Header.Set("User-Agent", userAgent)
+	res, err := c.hc.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return "", "", NotDefinedError(suffix)
+	}
+	if res.StatusCode != 200 {
+		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
+	}
+	all, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", err
+	}
+	return string(all), res.Header.Get("Etag"), nil
+}
+
+// Get returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+func (c *Client) Get(suffix string) (string, error) {
+	val, _, err := c.getETag(suffix)
+	return val, err
+}
+
+func (c *Client) getTrimmed(suffix string) (s string, err error) {
+	s, err = c.Get(suffix)
+	s = strings.TrimSpace(s)
+	return
+}
+
+func (c *Client) lines(suffix string) ([]string, error) {
+	j, err := c.Get(suffix)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.Split(strings.TrimSpace(j), "\n")
+	for i := range s {
+		s[i] = strings.TrimSpace(s[i])
+	}
+	return s, nil
+}
+
+// ProjectID returns the current instance's project ID string.
+func (c *Client) ProjectID() (string, error) { return projID.get(c) }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) }
+
+// InstanceID returns the current VM's numeric instance ID.
+func (c *Client) InstanceID() (string, error) { return instID.get(c) }
+
+// InternalIP returns the instance's primary internal IP address.
+func (c *Client) InternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/ip")
+}
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func (c *Client) ExternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+}
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func (c *Client) Hostname() (string, error) {
+	return c.getTrimmed("instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func (c *Client) InstanceTags() ([]string, error) {
+	var s []string
+	j, err := c.Get("instance/tags")
+	if err != nil {
+		return nil, err
+	}
+	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// InstanceName returns the current VM's instance ID string.
+func (c *Client) InstanceName() (string, error) {
+	host, err := c.Hostname()
+	if err != nil {
+		return "", err
+	}
+	return strings.Split(host, ".")[0], nil
+}
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func (c *Client) Zone() (string, error) {
+	zone, err := c.getTrimmed("instance/zone")
+	// zone is of the form "projects/<projNum>/zones/<zoneName>".
+	if err != nil {
+		return "", err
+	}
+	return zone[strings.LastIndex(zone, "/")+1:], nil
+}
+
+// InstanceAttributes returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") }
+
+// ProjectAttributes returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") }
+
+// InstanceAttributeValue returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) InstanceAttributeValue(attr string) (string, error) {
+	return c.Get("instance/attributes/" + attr)
+}
+
+// ProjectAttributeValue returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) ProjectAttributeValue(attr string) (string, error) {
+	return c.Get("project/attributes/" + attr)
+}
+
+// Scopes returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) Scopes(serviceAccount string) ([]string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return c.lines("instance/service-accounts/" + serviceAccount + "/scopes")
+}
+
 // Subscribe subscribes to a value from the metadata service.
 // The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
 // The suffix may contain query parameters.
@@ -275,11 +464,11 @@ func systemInfoSuggestsGCE() bool {
 // and ok false. Subscribe blocks until fn returns a non-nil error or the value
 // is deleted. Subscribe returns the error value returned from the last call to
 // fn, which may be nil when ok == false.
-func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error {
 	const failedSubscribeSleep = time.Second * 5
 
 	// First check to see if the metadata value exists at all.
-	val, lastETag, err := getETag(subscribeClient, suffix)
+	val, lastETag, err := c.getETag(suffix)
 	if err != nil {
 		return err
 	}
@@ -295,7 +484,7 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 		suffix += "?wait_for_change=true&last_etag="
 	}
 	for {
-		val, etag, err := getETag(subscribeClient, suffix+url.QueryEscape(lastETag))
+		val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag))
 		if err != nil {
 			if _, deleted := err.(NotDefinedError); !deleted {
 				time.Sleep(failedSubscribeSleep)
@@ -310,128 +499,3 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 		}
 	}
 }
-
-// ProjectID returns the current instance's project ID string.
-func ProjectID() (string, error) { return projID.get() }
-
-// NumericProjectID returns the current instance's numeric project ID.
-func NumericProjectID() (string, error) { return projNum.get() }
-
-// InternalIP returns the instance's primary internal IP address.
-func InternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/ip")
-}
-
-// ExternalIP returns the instance's primary external (public) IP address.
-func ExternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
-}
-
-// Hostname returns the instance's hostname. This will be of the form
-// "<instanceID>.c.<projID>.internal".
-func Hostname() (string, error) {
-	return getTrimmed("instance/hostname")
-}
-
-// InstanceTags returns the list of user-defined instance tags,
-// assigned when initially creating a GCE instance.
-func InstanceTags() ([]string, error) {
-	var s []string
-	j, err := Get("instance/tags")
-	if err != nil {
-		return nil, err
-	}
-	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
-		return nil, err
-	}
-	return s, nil
-}
-
-// InstanceID returns the current VM's numeric instance ID.
-func InstanceID() (string, error) {
-	return instID.get()
-}
-
-// InstanceName returns the current VM's instance ID string.
-func InstanceName() (string, error) {
-	host, err := Hostname()
-	if err != nil {
-		return "", err
-	}
-	return strings.Split(host, ".")[0], nil
-}
-
-// Zone returns the current VM's zone, such as "us-central1-b".
-func Zone() (string, error) {
-	zone, err := getTrimmed("instance/zone")
-	// zone is of the form "projects/<projNum>/zones/<zoneName>".
-	if err != nil {
-		return "", err
-	}
-	return zone[strings.LastIndex(zone, "/")+1:], nil
-}
-
-// InstanceAttributes returns the list of user-defined attributes,
-// assigned when initially creating a GCE VM instance. The value of an
-// attribute can be obtained with InstanceAttributeValue.
-func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }
-
-// ProjectAttributes returns the list of user-defined attributes
-// applying to the project as a whole, not just this VM.  The value of
-// an attribute can be obtained with ProjectAttributeValue.
-func ProjectAttributes() ([]string, error) { return lines("project/attributes/") }
-
-func lines(suffix string) ([]string, error) {
-	j, err := Get(suffix)
-	if err != nil {
-		return nil, err
-	}
-	s := strings.Split(strings.TrimSpace(j), "\n")
-	for i := range s {
-		s[i] = strings.TrimSpace(s[i])
-	}
-	return s, nil
-}
-
-// InstanceAttributeValue returns the value of the provided VM
-// instance attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// InstanceAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func InstanceAttributeValue(attr string) (string, error) {
-	return Get("instance/attributes/" + attr)
-}
-
-// ProjectAttributeValue returns the value of the provided
-// project attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// ProjectAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func ProjectAttributeValue(attr string) (string, error) {
-	return Get("project/attributes/" + attr)
-}
-
-// Scopes returns the service account scopes for the given account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
-func Scopes(serviceAccount string) ([]string, error) {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return lines("instance/service-accounts/" + serviceAccount + "/scopes")
-}
-
-func strsContains(ss []string, s string) bool {
-	for _, v := range ss {
-		if v == s {
-			return true
-		}
-	}
-	return false
-}

backend/vendor/github.com/davecgh/go-spew/LICENSE

@@ -2,7 +2,7 @@ ISC License
 
 Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
 
-Permission to use, copy, modify, and distribute this software for any
+Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
 copyright notice and this permission notice appear in all copies.
 

backend/vendor/github.com/davecgh/go-spew/spew/bypass.go

@@ -16,7 +16,9 @@
 // when the code is not running on Google App Engine, compiled by GopherJS, and
 // "-tags safe" is not added to the go build command line.  The "disableunsafe"
 // tag is deprecated and thus should not be used.
-// +build !js,!appengine,!safe,!disableunsafe
+// Go versions prior to 1.4 are disabled because they use a different layout
+// for interfaces which make the implementation of unsafeReflectValue more complex.
+// +build !js,!appengine,!safe,!disableunsafe,go1.4
 
 package spew
 
@@ -34,80 +36,49 @@ const (
 	ptrSize = unsafe.Sizeof((*byte)(nil))
 )
 
-var (
-	// offsetPtr, offsetScalar, and offsetFlag are the offsets for the
-	// internal reflect.Value fields.  These values are valid before golang
-	// commit ecccf07e7f9d which changed the format.  The are also valid
-	// after commit 82f48826c6c7 which changed the format again to mirror
-	// the original format.  Code in the init function updates these offsets
-	// as necessary.
-	offsetPtr    = uintptr(ptrSize)
-	offsetScalar = uintptr(0)
-	offsetFlag   = uintptr(ptrSize * 2)
+type flag uintptr
 
-	// flagKindWidth and flagKindShift indicate various bits that the
-	// reflect package uses internally to track kind information.
-	//
-	// flagRO indicates whether or not the value field of a reflect.Value is
-	// read-only.
-	//
-	// flagIndir indicates whether the value field of a reflect.Value is
-	// the actual data or a pointer to the data.
-	//
-	// These values are valid before golang commit 90a7c3c86944 which
-	// changed their positions.  Code in the init function updates these
-	// flags as necessary.
-	flagKindWidth = uintptr(5)
-	flagKindShift = uintptr(flagKindWidth - 1)
-	flagRO        = uintptr(1 << 0)
-	flagIndir     = uintptr(1 << 1)
+var (
+	// flagRO indicates whether the value field of a reflect.Value
+	// is read-only.
+	flagRO flag
+
+	// flagAddr indicates whether the address of the reflect.Value's
+	// value may be taken.
+	flagAddr flag
 )
 
-func init() {
-	// Older versions of reflect.Value stored small integers directly in the
-	// ptr field (which is named val in the older versions).  Versions
-	// between commits ecccf07e7f9d and 82f48826c6c7 added a new field named
-	// scalar for this purpose which unfortunately came before the flag
-	// field, so the offset of the flag field is different for those
-	// versions.
-	//
-	// This code constructs a new reflect.Value from a known small integer
-	// and checks if the size of the reflect.Value struct indicates it has
-	// the scalar field. When it does, the offsets are updated accordingly.
-	vv := reflect.ValueOf(0xf00)
-	if unsafe.Sizeof(vv) == (ptrSize * 4) {
-		offsetScalar = ptrSize * 2
-		offsetFlag = ptrSize * 3
-	}
+// flagKindMask holds the bits that make up the kind
+// part of the flags field. In all the supported versions,
+// it is in the lower 5 bits.
+const flagKindMask = flag(0x1f)
 
-	// Commit 90a7c3c86944 changed the flag positions such that the low
-	// order bits are the kind.  This code extracts the kind from the flags
-	// field and ensures it's the correct type.  When it's not, the flag
-	// order has been changed to the newer format, so the flags are updated
-	// accordingly.
-	upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag)
-	upfv := *(*uintptr)(upf)
-	flagKindMask := uintptr((1<<flagKindWidth - 1) << flagKindShift)
-	if (upfv&flagKindMask)>>flagKindShift != uintptr(reflect.Int) {
-		flagKindShift = 0
-		flagRO = 1 << 5
-		flagIndir = 1 << 6
+// Different versions of Go have used different
+// bit layouts for the flags type. This table
+// records the known combinations.
+var okFlags = []struct {
+	ro, addr flag
+}{{
+	// From Go 1.4 to 1.5
+	ro:   1 << 5,
+	addr: 1 << 7,
+}, {
+	// Up to Go tip.
+	ro:   1<<5 | 1<<6,
+	addr: 1 << 8,
+}}
 
-		// Commit adf9b30e5594 modified the flags to separate the
-		// flagRO flag into two bits which specifies whether or not the
-		// field is embedded.  This causes flagIndir to move over a bit
-		// and means that flagRO is the combination of either of the
-		// original flagRO bit and the new bit.
-		//
-		// This code detects the change by extracting what used to be
-		// the indirect bit to ensure it's set.  When it's not, the flag
-		// order has been changed to the newer format, so the flags are
-		// updated accordingly.
-		if upfv&flagIndir == 0 {
-			flagRO = 3 << 5
-			flagIndir = 1 << 7
-		}
+var flagValOffset = func() uintptr {
+	field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
+	if !ok {
+		panic("reflect.Value has no flag field")
 	}
+	return field.Offset
+}()
+
+// flagField returns a pointer to the flag field of a reflect.Value.
+func flagField(v *reflect.Value) *flag {
+	return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
 }
 
 // unsafeReflectValue converts the passed reflect.Value into a one that bypasses
@@ -119,34 +90,56 @@ func init() {
 // This allows us to check for implementations of the Stringer and error
 // interfaces to be used for pretty printing ordinarily unaddressable and
 // inaccessible values such as unexported struct fields.
-func unsafeReflectValue(v reflect.Value) (rv reflect.Value) {
-	indirects := 1
-	vt := v.Type()
-	upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr)
-	rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag))
-	if rvf&flagIndir != 0 {
-		vt = reflect.PtrTo(v.Type())
-		indirects++
-	} else if offsetScalar != 0 {
-		// The value is in the scalar field when it's not one of the
-		// reference types.
-		switch vt.Kind() {
-		case reflect.Uintptr:
-		case reflect.Chan:
-		case reflect.Func:
-		case reflect.Map:
-		case reflect.Ptr:
-		case reflect.UnsafePointer:
-		default:
-			upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) +
-				offsetScalar)
+func unsafeReflectValue(v reflect.Value) reflect.Value {
+	if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
+		return v
+	}
+	flagFieldPtr := flagField(&v)
+	*flagFieldPtr &^= flagRO
+	*flagFieldPtr |= flagAddr
+	return v
+}
+
+// Sanity checks against future reflect package changes
+// to the type or semantics of the Value.flag field.
+func init() {
+	field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
+	if !ok {
+		panic("reflect.Value has no flag field")
+	}
+	if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
+		panic("reflect.Value flag field has changed kind")
+	}
+	type t0 int
+	var t struct {
+		A t0
+		// t0 will have flagEmbedRO set.
+		t0
+		// a will have flagStickyRO set
+		a t0
+	}
+	vA := reflect.ValueOf(t).FieldByName("A")
+	va := reflect.ValueOf(t).FieldByName("a")
+	vt0 := reflect.ValueOf(t).FieldByName("t0")
+
+	// Infer flagRO from the difference between the flags
+	// for the (otherwise identical) fields in t.
+	flagPublic := *flagField(&vA)
+	flagWithRO := *flagField(&va) | *flagField(&vt0)
+	flagRO = flagPublic ^ flagWithRO
+
+	// Infer flagAddr from the difference between a value
+	// taken from a pointer and not.
+	vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
+	flagNoPtr := *flagField(&vA)
+	flagPtr := *flagField(&vPtrA)
+	flagAddr = flagNoPtr ^ flagPtr
+
+	// Check that the inferred flags tally with one of the known versions.
+	for _, f := range okFlags {
+		if flagRO == f.ro && flagAddr == f.addr {
+			return
 		}
 	}
-
-	pv := reflect.NewAt(vt, upv)
-	rv = pv
-	for i := 0; i < indirects; i++ {
-		rv = rv.Elem()
-	}
-	return rv
+	panic("reflect.Value read-only flag has changed semantics")
 }

backend/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go

@@ -16,7 +16,7 @@
 // when the code is running on Google App Engine, compiled by GopherJS, or
 // "-tags safe" is added to the go build command line.  The "disableunsafe"
 // tag is deprecated and thus should not be used.
-// +build js appengine safe disableunsafe
+// +build js appengine safe disableunsafe !go1.4
 
 package spew
 

backend/vendor/github.com/davecgh/go-spew/spew/common.go

@@ -180,7 +180,7 @@ func printComplex(w io.Writer, c complex128, floatPrecision int) {
 	w.Write(closeParenBytes)
 }
 
-// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x'
+// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x'
 // prefix to Writer w.
 func printHexPtr(w io.Writer, p uintptr) {
 	// Null pointer.

backend/vendor/github.com/davecgh/go-spew/spew/dump.go

@@ -35,16 +35,16 @@ var (
 
 	// cCharRE is a regular expression that matches a cgo char.
 	// It is used to detect character arrays to hexdump them.
-	cCharRE = regexp.MustCompile("^.*\\._Ctype_char$")
+	cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`)
 
 	// cUnsignedCharRE is a regular expression that matches a cgo unsigned
 	// char.  It is used to detect unsigned character arrays to hexdump
 	// them.
-	cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$")
+	cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`)
 
 	// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
 	// It is used to detect uint8_t arrays to hexdump them.
-	cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$")
+	cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`)
 )
 
 // dumpState contains information about the state of a dump operation.
@@ -143,10 +143,10 @@ func (d *dumpState) dumpPtr(v reflect.Value) {
 	// Display dereferenced value.
 	d.w.Write(openParenBytes)
 	switch {
-	case nilFound == true:
+	case nilFound:
 		d.w.Write(nilAngleBytes)
 
-	case cycleFound == true:
+	case cycleFound:
 		d.w.Write(circularBytes)
 
 	default:

backend/vendor/github.com/davecgh/go-spew/spew/format.go

@@ -182,10 +182,10 @@ func (f *formatState) formatPtr(v reflect.Value) {
 
 	// Display dereferenced value.
 	switch {
-	case nilFound == true:
+	case nilFound:
 		f.fs.Write(nilAngleBytes)
 
-	case cycleFound == true:
+	case cycleFound:
 		f.fs.Write(circularShortBytes)
 
 	default:

backend/vendor/github.com/globalsign/mgo/.travis.yml

@@ -29,7 +29,7 @@ install:
     - go get gopkg.in/check.v1
     - go get gopkg.in/yaml.v2
     - go get gopkg.in/tomb.v2
-    - go get github.com/golang/lint
+    - go get golang.org/x/lint/golint
 
 before_script:
     - golint  ./... | grep -v 'ID' | cat

backend/vendor/github.com/globalsign/mgo/README.md

@@ -13,6 +13,12 @@ Detailed documentation of the API is available at
 
 A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver.
 
+## Supported Versions
+
+`mgo` is known to work well on (and has integration tests against) MongoDB v3.0, 3.2, 3.4 and 3.6. 
+
+MongoDB 4.0 is currently experimental - we would happily accept PRs to help improve support!
+
 ## Changes
 * Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254))
 * Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288))

backend/vendor/github.com/go-chi/chi/.travis.yml

@@ -1,18 +1,18 @@
 language: go
 
 go:
-  - 1.7.x
-  - 1.8.x
-  - 1.9.x
-
-install:
-  - go get -u golang.org/x/tools/cmd/goimports
-  - go get -u github.com/golang/lint/golint
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
 
 script:
   - go get -d -t ./...
   - go vet ./...
-  - golint ./...
   - go test ./...
   - >
-    goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :
+    go_version=$(go version);
+    if [ ${go_version:13:4} = "1.12" ]; then
+      go get -u golang.org/x/tools/cmd/goimports;
+      goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :;
+    fi
+

backend/vendor/github.com/go-chi/chi/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v4.0.0 (2019-01-10)
+
+- chi v4 requires Go 1.10.3+ (or Go 1.9.7+) - we have deprecated support for Go 1.7 and 1.8
+- router: respond with 404 on router with no routes (#362)
+- router: additional check to ensure wildcard is at the end of a url pattern (#333)
+- middleware: deprecate use of http.CloseNotifier (#347)
+- middleware: fix RedirectSlashes to include query params on redirect (#334)
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.4...v4.0.0
+
+
+## v3.3.4 (2019-01-07)
+
+- Minor middleware improvements. No changes to core library/router. Moving v3 into its
+- own branch as a version of chi for Go 1.7, 1.8, 1.9, 1.10, 1.11
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.3...v3.3.4
+
+
+## v3.3.3 (2018-08-27)
+
+- Minor release
+- See https://github.com/go-chi/chi/compare/v3.3.2...v3.3.3
+
+
 ## v3.3.2 (2017-12-22)
 
 - Support to route trailing slashes on mounted sub-routers (#281)

backend/vendor/github.com/go-chi/chi/README.md

@@ -3,7 +3,7 @@
 
 [![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis]
 
-`chi` is a lightweight, idiomatic and composable router for building Go 1.7+ HTTP services. It's
+`chi` is a lightweight, idiomatic and composable router for building Go HTTP services. It's
 especially good at helping you write large REST API services that are kept maintainable as your
 project grows and changes. `chi` is built on the new `context` package introduced in Go 1.7 to
 handle signaling, cancelation and request-scoped values across a handler chain.
@@ -31,18 +31,12 @@ included some useful/optional subpackages: [middleware](/middleware), [render](h
 * **Context control** - built on new `context` package, providing value chaining, cancelations and timeouts
 * **Robust** - in production at Pressly, CloudFlare, Heroku, 99Designs, and many others (see [discussion](https://github.com/go-chi/chi/issues/91))
 * **Doc generation** - `docgen` auto-generates routing documentation from your source to JSON or Markdown
-* **No external dependencies** - plain ol' Go 1.7+ stdlib + net/http
+* **No external dependencies** - plain ol' Go stdlib + net/http
 
 
 ## Examples
 
-* [rest](https://github.com/go-chi/chi/blob/master/_examples/rest/main.go) - REST APIs made easy, productive and maintainable
-* [logging](https://github.com/go-chi/chi/blob/master/_examples/logging/main.go) - Easy structured logging for any backend
-* [limits](https://github.com/go-chi/chi/blob/master/_examples/limits/main.go) - Timeouts and Throttling
-* [todos-resource](https://github.com/go-chi/chi/blob/master/_examples/todos-resource/main.go) - Struct routers/handlers, an example of another code layout style
-* [versions](https://github.com/go-chi/chi/blob/master/_examples/versions/main.go) - Demo of `chi/render` subpkg
-* [fileserver](https://github.com/go-chi/chi/blob/master/_examples/fileserver/main.go) - Easily serve static files
-* [graceful](https://github.com/go-chi/chi/blob/master/_examples/graceful/main.go) - Graceful context signaling and server shutdown
+See [_examples/](https://github.com/go-chi/chi/blob/master/_examples/) for a variety of examples.
 
 
 **As easy as:**
@@ -70,8 +64,8 @@ Here is a little preview of how routing looks like with chi. Also take a look at
 in JSON ([routes.json](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.json)) and in
 Markdown ([routes.md](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.md)).
 
-I highly recommend reading the source of the [examples](#examples) listed above, they will show you all the features
-of chi and serve as a good form of documentation.
+I highly recommend reading the source of the [examples](https://github.com/go-chi/chi/blob/master/_examples/) listed
+above, they will show you all the features of chi and serve as a good form of documentation.
 
 ```go
 import (
@@ -232,7 +226,7 @@ type Router interface {
 }
 
 // Routes interface adds two methods for router traversal, which is also
-// used by the `docgen` subpackage to generation documentation for Routers.
+// used by the github.com/go-chi/docgen package to generate documentation for Routers.
 type Routes interface {
 	// Routes returns the routing tree in an easily traversable structure.
 	Routes() []Route
@@ -261,7 +255,7 @@ friendly with any middleware in the community. This offers much better extensibi
 of packages and is at the heart of chi's purpose.
 
 Here is an example of a standard net/http middleware handler using the new request context
-available in Go 1.7+. This middleware sets a hypothetical user identifier on the request
+available in Go. This middleware sets a hypothetical user identifier on the request
 context and calls the next handler in the chain.
 
 ```go
@@ -347,6 +341,7 @@ Please see https://github.com/go-chi for additional packages.
 | package                                            | description                                                 |
 |:---------------------------------------------------|:-------------------------------------------------------------
 | [cors](https://github.com/go-chi/cors)             | Cross-origin resource sharing (CORS)                        |
+| [docgen](https://github.com/go-chi/docgen)         | Print chi.Router routes at runtime                          |
 | [jwtauth](https://github.com/go-chi/jwtauth)       | JWT authentication                                          |
 | [hostrouter](https://github.com/go-chi/hostrouter) | Domain/host based request routing                           |
 | [httpcoala](https://github.com/go-chi/httpcoala)   | HTTP request coalescer                                      |
@@ -374,33 +369,33 @@ and..
 
 The benchmark suite: https://github.com/pkieltyka/go-http-routing-benchmark
 
-Results as of Aug 31, 2017 on Go 1.9.0
+Results as of Jan 9, 2019 with Go 1.11.4 on Linux X1 Carbon laptop
 
 ```shell
-BenchmarkChi_Param        	 3000000	       607 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Param5       	 2000000	       935 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Param20      	 1000000	      1944 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParamWrite   	 2000000	       664 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubStatic 	 2000000	       627 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubParam  	 2000000	       847 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubAll    	   10000	    175556 ns/op	   87700 B/op	     609 allocs/op
-BenchmarkChi_GPlusStatic  	 3000000	       566 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlusParam   	 2000000	       652 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlus2Params 	 2000000	       767 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlusAll     	  200000	      9794 ns/op	    5616 B/op	      39 allocs/op
-BenchmarkChi_ParseStatic  	 3000000	       590 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParseParam   	 2000000	       656 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Parse2Params 	 2000000	       715 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParseAll     	  100000	     18045 ns/op	   11232 B/op	      78 allocs/op
-BenchmarkChi_StaticAll    	   10000	    108871 ns/op	   67827 B/op	     471 allocs/op
+BenchmarkChi_Param            3000000         475 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Param5           2000000         696 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Param20          1000000        1275 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParamWrite       3000000         505 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubStatic     3000000         508 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubParam      2000000         669 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubAll          10000      134627 ns/op     87699 B/op    609 allocs/op
+BenchmarkChi_GPlusStatic      3000000         402 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlusParam       3000000         500 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlus2Params     3000000         586 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlusAll          200000        7237 ns/op      5616 B/op     39 allocs/op
+BenchmarkChi_ParseStatic      3000000         408 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParseParam       3000000         488 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Parse2Params     3000000         551 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParseAll          100000       13508 ns/op     11232 B/op     78 allocs/op
+BenchmarkChi_StaticAll          20000       81933 ns/op     67826 B/op    471 allocs/op
 ```
 
-Comparison with other routers: https://gist.github.com/pkieltyka/c089f309abeb179cfc4deaa519956d8c
+Comparison with other routers: https://gist.github.com/pkieltyka/123032f12052520aaccab752bd3e78cc
 
 NOTE: the allocs in the benchmark above are from the calls to http.Request's
 `WithContext(context.Context)` method that clones the http.Request, sets the `Context()`
 on the duplicated (alloc'd) request and returns it the new request object. This is just
-how setting context on a request in Go 1.7+ works.
+how setting context on a request in Go works.
 
 
 ## Credits

backend/vendor/github.com/go-chi/chi/context.go

@@ -84,13 +84,13 @@ func (x *Context) URLParam(key string) string {
 //
 // For example,
 //
-// func Instrument(next http.Handler) http.Handler {
-//   return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-//     next.ServeHTTP(w, r)
-//     routePattern := chi.RouteContext(r.Context()).RoutePattern()
-//     measure(w, r, routePattern)
-// 	 })
-// }
+//   func Instrument(next http.Handler) http.Handler {
+//     return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+//       next.ServeHTTP(w, r)
+//       routePattern := chi.RouteContext(r.Context()).RoutePattern()
+//       measure(w, r, routePattern)
+//   	 })
+//   }
 func (x *Context) RoutePattern() string {
 	routePattern := strings.Join(x.RoutePatterns, "")
 	return strings.Replace(routePattern, "/*/", "/", -1)

backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go

@@ -1,42 +0,0 @@
-// +build go1.7,!go1.8
-
-package middleware
-
-import (
-	"context"
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		cn, ok := w.(http.CloseNotifier)
-		if !ok {
-			panic("chi/middleware: CloseNotify expects http.ResponseWriter to implement http.CloseNotifier interface")
-		}
-		closeNotifyCh := cn.CloseNotify()
-
-		ctx, cancel := context.WithCancel(r.Context())
-		defer cancel()
-
-		go func() {
-			select {
-			case <-ctx.Done():
-				return
-			case <-closeNotifyCh:
-				cancel()
-				return
-			}
-		}()
-
-		r = r.WithContext(ctx)
-		next.ServeHTTP(w, r)
-	}
-
-	return http.HandlerFunc(fn)
-}

backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go

@@ -1,17 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	return next
-}

backend/vendor/github.com/go-chi/chi/middleware/compress.go

@@ -11,24 +11,98 @@ import (
 	"strings"
 )
 
-type encoding int
+var encoders = map[string]EncoderFunc{}
 
-const (
-	encodingNone encoding = iota
-	encodingGzip
-	encodingDeflate
-)
+var encodingPrecedence = []string{"br", "gzip", "deflate"}
+
+func init() {
+	// TODO:
+	// lzma: Opera.
+	// sdch: Chrome, Android. Gzip output + dictionary header.
+	// br:   Brotli, see https://github.com/go-chi/chi/pull/326
+
+	// TODO: Exception for old MSIE browsers that can't handle non-HTML?
+	// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	SetEncoder("gzip", encoderGzip)
+
+	// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
+	// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
+	// checksum compared to CRC-32 used in "gzip" and thus is faster.
+	//
+	// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
+	// raw DEFLATE data only, without the mentioned zlib wrapper.
+	// Because of this major confusion, most modern browsers try it
+	// both ways, first looking for zlib headers.
+	// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
+	//
+	// The list of browsers having problems is quite big, see:
+	// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
+	//
+	// That's why we prefer gzip over deflate. It's just more reliable
+	// and not significantly slower than gzip.
+	SetEncoder("deflate", encoderDeflate)
+
+	// NOTE: Not implemented, intentionally:
+	// case "compress": // LZW. Deprecated.
+	// case "bzip2":    // Too slow on-the-fly.
+	// case "zopfli":   // Too slow on-the-fly.
+	// case "xz":       // Too slow on-the-fly.
+}
+
+// An EncoderFunc is a function that wraps the provided ResponseWriter with a
+// streaming compression algorithm and returns it.
+//
+// In case of failure, the function should return nil.
+type EncoderFunc func(w http.ResponseWriter, level int) io.Writer
+
+// SetEncoder can be used to set the implementation of a compression algorithm.
+//
+// The encoding should be a standardised identifier. See:
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
+//
+// For example, add the Brotli algortithm:
+//
+//  import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc"
+//
+//  middleware.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer {
+//    params := brotli_enc.NewBrotliParams()
+//    params.SetQuality(level)
+//    return brotli_enc.NewBrotliWriter(params, w)
+//  })
+func SetEncoder(encoding string, fn EncoderFunc) {
+	encoding = strings.ToLower(encoding)
+	if encoding == "" {
+		panic("the encoding can not be empty")
+	}
+	if fn == nil {
+		panic("attempted to set a nil encoder function")
+	}
+	encoders[encoding] = fn
+
+	var e string
+	for _, v := range encodingPrecedence {
+		if v == encoding {
+			e = v
+		}
+	}
+
+	if e == "" {
+		encodingPrecedence = append([]string{e}, encodingPrecedence...)
+	}
+}
 
 var defaultContentTypes = map[string]struct{}{
-	"text/html":                struct{}{},
-	"text/css":                 struct{}{},
-	"text/plain":               struct{}{},
-	"text/javascript":          struct{}{},
-	"application/javascript":   struct{}{},
-	"application/x-javascript": struct{}{},
-	"application/json":         struct{}{},
-	"application/atom+xml":     struct{}{},
-	"application/rss+xml":      struct{}{},
+	"text/html":                {},
+	"text/css":                 {},
+	"text/plain":               {},
+	"text/javascript":          {},
+	"application/javascript":   {},
+	"application/x-javascript": {},
+	"application/json":         {},
+	"application/atom+xml":     {},
+	"application/rss+xml":      {},
+	"image/svg+xml":            {},
 }
 
 // DefaultCompress is a middleware that compresses response
@@ -43,6 +117,11 @@ func DefaultCompress(next http.Handler) http.Handler {
 // body of a given content types to a data format based
 // on Accept-Encoding request header. It uses a given
 // compression level.
+//
+// NOTE: make sure to set the Content-Type header on your response
+// otherwise this middleware will not compress the response body. For ex, in
+// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody))
+// or set it manually.
 func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 	contentTypes := defaultContentTypes
 	if len(types) > 0 {
@@ -54,159 +133,143 @@ func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
-			mcw := &maybeCompressResponseWriter{
+			encoder, encoding := selectEncoder(r.Header)
+
+			cw := &compressResponseWriter{
 				ResponseWriter: w,
 				w:              w,
 				contentTypes:   contentTypes,
-				encoding:       selectEncoding(r.Header),
+				encoder:        encoder,
+				encoding:       encoding,
 				level:          level,
 			}
-			defer mcw.Close()
+			defer cw.Close()
 
-			next.ServeHTTP(mcw, r)
+			next.ServeHTTP(cw, r)
 		}
 
 		return http.HandlerFunc(fn)
 	}
 }
 
-func selectEncoding(h http.Header) encoding {
-	enc := h.Get("Accept-Encoding")
+func selectEncoder(h http.Header) (EncoderFunc, string) {
+	header := h.Get("Accept-Encoding")
 
-	switch {
-	// TODO:
-	// case "br":    // Brotli, experimental. Firefox 2016, to-be-in Chromium.
-	// case "lzma":  // Opera.
-	// case "sdch":  // Chrome, Android. Gzip output + dictionary header.
+	// Parse the names of all accepted algorithms from the header.
+	accepted := strings.Split(strings.ToLower(header), ",")
 
-	case strings.Contains(enc, "gzip"):
-		// TODO: Exception for old MSIE browsers that can't handle non-HTML?
-		// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
-		return encodingGzip
-
-	case strings.Contains(enc, "deflate"):
-		// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
-		// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
-		// checksum compared to CRC-32 used in "gzip" and thus is faster.
-		//
-		// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
-		// raw DEFLATE data only, without the mentioned zlib wrapper.
-		// Because of this major confusion, most modern browsers try it
-		// both ways, first looking for zlib headers.
-		// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
-		//
-		// The list of browsers having problems is quite big, see:
-		// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
-		// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
-		//
-		// That's why we prefer gzip over deflate. It's just more reliable
-		// and not significantly slower than gzip.
-		return encodingDeflate
-
-		// NOTE: Not implemented, intentionally:
-		// case "compress": // LZW. Deprecated.
-		// case "bzip2":    // Too slow on-the-fly.
-		// case "zopfli":   // Too slow on-the-fly.
-		// case "xz":       // Too slow on-the-fly.
+	// Find supported encoder by accepted list by precedence
+	for _, name := range encodingPrecedence {
+		if fn, ok := encoders[name]; ok && matchAcceptEncoding(accepted, name) {
+			return fn, name
+		}
 	}
 
-	return encodingNone
+	// No encoder found to match the accepted encoding
+	return nil, ""
 }
 
-type maybeCompressResponseWriter struct {
+func matchAcceptEncoding(accepted []string, encoding string) bool {
+	for _, v := range accepted {
+		if strings.Index(v, encoding) >= 0 {
+			return true
+		}
+	}
+	return false
+}
+
+type compressResponseWriter struct {
 	http.ResponseWriter
 	w            io.Writer
-	encoding     encoding
+	encoder      EncoderFunc
+	encoding     string
 	contentTypes map[string]struct{}
 	level        int
 	wroteHeader  bool
 }
 
-func (w *maybeCompressResponseWriter) WriteHeader(code int) {
-	if w.wroteHeader {
+func (cw *compressResponseWriter) WriteHeader(code int) {
+	if cw.wroteHeader {
 		return
 	}
-	w.wroteHeader = true
-	defer w.ResponseWriter.WriteHeader(code)
+	cw.wroteHeader = true
+	defer cw.ResponseWriter.WriteHeader(code)
 
 	// Already compressed data?
-	if w.ResponseWriter.Header().Get("Content-Encoding") != "" {
+	if cw.Header().Get("Content-Encoding") != "" {
 		return
 	}
-	// The content-length after compression is unknown
-	w.ResponseWriter.Header().Del("Content-Length")
 
 	// Parse the first part of the Content-Type response header.
 	contentType := ""
-	parts := strings.Split(w.ResponseWriter.Header().Get("Content-Type"), ";")
+	parts := strings.Split(cw.Header().Get("Content-Type"), ";")
 	if len(parts) > 0 {
 		contentType = parts[0]
 	}
 
 	// Is the content type compressable?
-	if _, ok := w.contentTypes[contentType]; !ok {
+	if _, ok := cw.contentTypes[contentType]; !ok {
 		return
 	}
 
-	// Select the compress writer.
-	switch w.encoding {
-	case encodingGzip:
-		gw, err := gzip.NewWriterLevel(w.ResponseWriter, w.level)
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
-		}
-		w.w = gw
-		w.ResponseWriter.Header().Set("Content-Encoding", "gzip")
+	if cw.encoder != nil && cw.encoding != "" {
+		if wr := cw.encoder(cw.ResponseWriter, cw.level); wr != nil {
+			cw.w = wr
+			cw.Header().Set("Content-Encoding", cw.encoding)
 
-	case encodingDeflate:
-		dw, err := flate.NewWriter(w.ResponseWriter, w.level)
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
+			// The content-length after compression is unknown
+			cw.Header().Del("Content-Length")
 		}
-		w.w = dw
-		w.ResponseWriter.Header().Set("Content-Encoding", "deflate")
 	}
 }
 
-func (w *maybeCompressResponseWriter) Write(p []byte) (int, error) {
-	if !w.wroteHeader {
-		w.WriteHeader(http.StatusOK)
+func (cw *compressResponseWriter) Write(p []byte) (int, error) {
+	if !cw.wroteHeader {
+		cw.WriteHeader(http.StatusOK)
 	}
 
-	return w.w.Write(p)
+	return cw.w.Write(p)
 }
 
-func (w *maybeCompressResponseWriter) Flush() {
-	if f, ok := w.w.(http.Flusher); ok {
+func (cw *compressResponseWriter) Flush() {
+	if f, ok := cw.w.(http.Flusher); ok {
 		f.Flush()
 	}
 }
 
-func (w *maybeCompressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	if hj, ok := w.w.(http.Hijacker); ok {
+func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	if hj, ok := cw.w.(http.Hijacker); ok {
 		return hj.Hijack()
 	}
 	return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer")
 }
 
-func (w *maybeCompressResponseWriter) CloseNotify() <-chan bool {
-	if cn, ok := w.w.(http.CloseNotifier); ok {
-		return cn.CloseNotify()
+func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
+	if ps, ok := cw.w.(http.Pusher); ok {
+		return ps.Push(target, opts)
 	}
-
-	// If the underlying writer does not implement http.CloseNotifier, return
-	// a channel that never receives a value. The semantics here is that the
-	// client never disconnnects before the request is processed by the
-	// http.Handler, which is close enough to the default behavior (when
-	// CloseNotify() is not even called).
-	return make(chan bool, 1)
+	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
 }
 
-func (w *maybeCompressResponseWriter) Close() error {
-	if c, ok := w.w.(io.WriteCloser); ok {
+func (cw *compressResponseWriter) Close() error {
+	if c, ok := cw.w.(io.WriteCloser); ok {
 		return c.Close()
 	}
 	return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer")
 }
+
+func encoderGzip(w http.ResponseWriter, level int) io.Writer {
+	gw, err := gzip.NewWriterLevel(w, level)
+	if err != nil {
+		return nil
+	}
+	return gw
+}
+
+func encoderDeflate(w http.ResponseWriter, level int) io.Writer {
+	dw, err := flate.NewWriter(w, level)
+	if err != nil {
+		return nil
+	}
+	return dw
+}

backend/vendor/github.com/go-chi/chi/middleware/compress18.go

@@ -1,15 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"errors"
-	"net/http"
-)
-
-func (w *maybeCompressResponseWriter) Push(target string, opts *http.PushOptions) error {
-	if ps, ok := w.w.(http.Pusher); ok {
-		return ps.Push(target, opts)
-	}
-	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
-}

backend/vendor/github.com/go-chi/chi/middleware/content_type.go

@@ -26,6 +26,12 @@ func AllowContentType(contentTypes ...string) func(next http.Handler) http.Handl
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
+			if r.ContentLength == 0 {
+				// skip check for empty content body
+				next.ServeHTTP(w, r)
+				return
+			}
+
 			s := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type")))
 			if i := strings.Index(s, ";"); i > -1 {
 				s = s[0:i]

backend/vendor/github.com/go-chi/chi/middleware/logger.go

@@ -16,7 +16,7 @@ var (
 	// DefaultLogger is called by the Logger middleware handler to log each request.
 	// Its made a package-level variable so that it can be reconfigured for custom
 	// logging configurations.
-	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags)})
+	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags), NoColor: false})
 )
 
 // Logger is a middleware that logs the start and end of each request, along
@@ -81,29 +81,32 @@ type LoggerInterface interface {
 
 // DefaultLogFormatter is a simple logger that implements a LogFormatter.
 type DefaultLogFormatter struct {
-	Logger LoggerInterface
+	Logger  LoggerInterface
+	NoColor bool
 }
 
 // NewLogEntry creates a new LogEntry for the request.
 func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
+	useColor := !l.NoColor
 	entry := &defaultLogEntry{
 		DefaultLogFormatter: l,
 		request:             r,
 		buf:                 &bytes.Buffer{},
+		useColor:            useColor,
 	}
 
 	reqID := GetReqID(r.Context())
 	if reqID != "" {
-		cW(entry.buf, nYellow, "[%s] ", reqID)
+		cW(entry.buf, useColor, nYellow, "[%s] ", reqID)
 	}
-	cW(entry.buf, nCyan, "\"")
-	cW(entry.buf, bMagenta, "%s ", r.Method)
+	cW(entry.buf, useColor, nCyan, "\"")
+	cW(entry.buf, useColor, bMagenta, "%s ", r.Method)
 
 	scheme := "http"
 	if r.TLS != nil {
 		scheme = "https"
 	}
-	cW(entry.buf, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
+	cW(entry.buf, useColor, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
 
 	entry.buf.WriteString("from ")
 	entry.buf.WriteString(r.RemoteAddr)
@@ -114,33 +117,34 @@ func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
 
 type defaultLogEntry struct {
 	*DefaultLogFormatter
-	request *http.Request
-	buf     *bytes.Buffer
+	request  *http.Request
+	buf      *bytes.Buffer
+	useColor bool
 }
 
 func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 	switch {
 	case status < 200:
-		cW(l.buf, bBlue, "%03d", status)
+		cW(l.buf, l.useColor, bBlue, "%03d", status)
 	case status < 300:
-		cW(l.buf, bGreen, "%03d", status)
+		cW(l.buf, l.useColor, bGreen, "%03d", status)
 	case status < 400:
-		cW(l.buf, bCyan, "%03d", status)
+		cW(l.buf, l.useColor, bCyan, "%03d", status)
 	case status < 500:
-		cW(l.buf, bYellow, "%03d", status)
+		cW(l.buf, l.useColor, bYellow, "%03d", status)
 	default:
-		cW(l.buf, bRed, "%03d", status)
+		cW(l.buf, l.useColor, bRed, "%03d", status)
 	}
 
-	cW(l.buf, bBlue, " %dB", bytes)
+	cW(l.buf, l.useColor, bBlue, " %dB", bytes)
 
 	l.buf.WriteString(" in ")
 	if elapsed < 500*time.Millisecond {
-		cW(l.buf, nGreen, "%s", elapsed)
+		cW(l.buf, l.useColor, nGreen, "%s", elapsed)
 	} else if elapsed < 5*time.Second {
-		cW(l.buf, nYellow, "%s", elapsed)
+		cW(l.buf, l.useColor, nYellow, "%s", elapsed)
 	} else {
-		cW(l.buf, nRed, "%s", elapsed)
+		cW(l.buf, l.useColor, nRed, "%s", elapsed)
 	}
 
 	l.Logger.Print(l.buf.String())
@@ -148,7 +152,7 @@ func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 
 func (l *defaultLogEntry) Panic(v interface{}, stack []byte) {
 	panicEntry := l.NewLogEntry(l.request).(*defaultLogEntry)
-	cW(panicEntry.buf, bRed, "panic: %+v", v)
+	cW(panicEntry.buf, l.useColor, bRed, "panic: %+v", v)
 	l.Logger.Print(panicEntry.buf.String())
 	l.Logger.Print(string(stack))
 }

backend/vendor/github.com/go-chi/chi/middleware/nocache.go

@@ -14,7 +14,7 @@ var epoch = time.Unix(0, 0).Format(time.RFC1123)
 // Taken from https://github.com/mytrile/nocache
 var noCacheHeaders = map[string]string{
 	"Expires":         epoch,
-	"Cache-Control":   "no-cache, no-store, must-revalidate, private, max-age=0",
+	"Cache-Control":   "no-cache, no-store, no-transform, must-revalidate, private, max-age=0",
 	"Pragma":          "no-cache",
 	"X-Accel-Expires": "0",
 }