Define remark host in runtime (#325)

* convert closest-polyfill to ts

* make remark respect host property in client setup

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
-/web/public/
+/frontend/node_modules/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,5 +19,5 @@ debug
 debug.test
 *.prof
 *.test
-/bin/
 remark42
+/backend/var/

.drone.yml

@@ -5,7 +5,7 @@ workspace:
 pipeline:
   
   build:
-    image: golang:1.10-alpine
+    image: golang:1.12-alpine
     commands:
       - sleep 5
       - nslookup mongo
@@ -99,4 +99,3 @@ services:
   mongo:
     image: mongo:3.6
     command: [ --smallfiles ]
-

.gitignore

@@ -8,13 +8,14 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
-/web/public/
-/web/coverage
+/frontend/node_modules/
+/frontend/public/
+/frontend/coverage
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
 .mongo
 remark42
-/bin/
+/bin/
+/backend/var/

Dockerfile

@@ -21,15 +21,19 @@ ARG DRONE_PULL_REQUEST
 ARG SKIP_BACKEND_TEST
 ARG MONGO_TEST
 
-WORKDIR /go/src/github.com/umputun/remark/backend
-ADD backend /go/src/github.com/umputun/remark/backend
-ADD .git /go/src/github.com/umputun/remark/.git
+ADD backend /build/backend
+ADD .git /build/.git
+WORKDIR /build/backend
+
+ENV GOFLAGS="-mod=vendor"
 
 # run tests
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
 RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
@@ -37,26 +41,20 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
-# coverage report
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    mkdir -p target && /script/coverage.sh ; \
-    else echo "skip backend coverage" ; fi
-
 # submit coverage to coverals if COVERALLS_TOKEN in env
 RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
     echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
@@ -65,11 +63,12 @@ RUN \
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
@@ -77,9 +76,9 @@ ARG CI
 ARG SKIP_FRONTEND_TEST
 ARG NODE_ENV=production
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
     if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
@@ -95,8 +94,8 @@ ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 

Dockerfile.artifacts

@@ -9,19 +9,20 @@ ARG DRONE_BRANCH
 ENV SKIP_FRONTEND_TEST=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
 ARG CI
 ARG NODE_ENV=production
 ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
-ADD web /srv/web
-RUN cd /srv/web && \
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
@@ -35,7 +36,7 @@ WORKDIR /go/src/github.com/umputun/remark/backend
 ADD backend /go/src/github.com/umputun/remark/backend
 ADD README.md /go/src/github.com/umputun/remark/
 ADD LICENSE /go/src/github.com/umputun/remark/
-COPY --from=build-frontend /srv/web/public/ web
+COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
     export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \

README.md

@@ -3,19 +3,22 @@
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
 * Social login via Google, Facebook, Github and Yandex
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
 * Import from disqus and wordpress
-* Markdown support
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
+* Telegram notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
 * Self-contained executable can be deployed directly to Linux, Windows and MacOS
-* Clean, lightweight and fully customizable UI
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
@@ -25,10 +28,11 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 
   - [Install](#install)
     - [Backend](#backend)
-      - [With Docker](#with-docker) 
-      - [Without docker](#without-docker) 
+      - [With Docker](#with-docker)
+      - [Without docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
+      - [Quick installation test](#quick-installation-test)
       - [Register oauth2 providers](#register-oauth2-providers)
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
@@ -70,73 +74,83 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
+* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
 * pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
 #### Without docker
 
 * download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
-* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows) 
+* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
 * run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
 * alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                      |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -144,7 +158,7 @@ _this is the recommended way to run remark42_
 
 ##### Required parameters
 
-Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
+Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
 2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
@@ -167,9 +181,13 @@ services:
             - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
             - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
         volumes:
-            - ./var:/srv/var                        # persistent volume to store all remark42 data 
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
 ```
 
+#### Quick installation test
+
+To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
+
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
@@ -222,6 +240,13 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 
 For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
 
+##### Anonymous Auth Provider
+
+Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
+
+- name should be at least 3 characters long
+- name has to start from the letter and contains letters, numbers, underscores and spaces only.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
@@ -230,7 +255,7 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co
 
 #### Initial import from WordPress
 
-1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 
+1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
 2. Move this file to your remark42 host within `./var`
 3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
 
@@ -258,43 +283,53 @@ Restore will clean all comments first and then will processed with complete impo
 ##### Backup format
 
 Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
-unmarshaled from `Comment` struct (see below). 
+unmarshaled from `Comment` struct (see below).
 
 #### Admin users
 
-Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line. 
+Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line.
 
 ```
     environment:
         - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
-To get user id just login and click on your username or any other user you want to promote to admins. 
+To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
 ### Setup on your website
 
 #### Comments
 
-It's a main widget which renders list of comments. 
+It's a main widget which renders list of comments.
 
 Add this snippet to the bottom of web page:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['embed'] // optional param; which components to load. default to ["embed"]
+                          // to load all components define components as ['embed', 'last-comments', 'counter']
+                          // available component are:
+                          //     - 'embed': basic comments widget
+                          //     - 'last-comments': last comments widget, see `Last Comments` section below
+                          //     - 'counter': counter widget, see `Counter` section below
     url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -302,7 +337,7 @@ And then add this node in the place where you want to see Remark42 widget:
 
 ```html
 <div id="remark42"></div>
-``` 
+```
 
 After that widget will be rendered inside this node.
 
@@ -311,31 +346,36 @@ After that widget will be rendered inside this node.
 Right now Remark has two themes: light and dark.
 You can pick one using configuration object,
 but there is also a possibility to switch between themes in runtime.
-For this purpose Remark adds to `window` object named `REMARK42`, 
+For this purpose Remark adds to `window` object named `REMARK42`,
 which contains function `changeTheme`.
 Just call this function and pass a name of the theme that you want to turn on:
 
 ```js
 window.REMARK42.changeTheme('light');
-```      
+```
 
 #### Last comments
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
+    site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -351,19 +391,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -373,18 +418,18 @@ And then add a node like this in the place where you want to see a number of com
 <span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
 ```
 
-You can use as many nodes like this as you need to. 
-The script will found all them by the class `remark__counter`, 
+You can use as many nodes like this as you need to.
+The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
+Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined.
 
 ## Build from the source
 
 - to build docker container - `make docker`. This command will produce container `umputun/remark42`.
-- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable 
+- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
  `remark42` file with everything embedded.
- 
+
 ## Development
 
 You can use fully functional local version to develop and test both frontend & backend.
@@ -402,11 +447,11 @@ docker-compose -f compose-dev-frontend.yml up
 
 It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
 To access UI demo page go to `127.0.0.1:8080/web`.
-By default, you would be logged in as `dev_user` which defined as admin. 
+By default, you would be logged in as `dev_user` which defined as admin.
 You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
 
-Backend docker compose config by default skips running frontend related tests. 
-Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build. 
+Backend docker compose config by default skips running frontend related tests.
+Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
 
 ### Backend development
 
@@ -425,9 +470,9 @@ To run backend with mongodb store mongo container should be started first - `doc
 
 * install [Node.js 8](https://nodejs.org/en/) or higher;
 * install [NPM 6.1.0](https://www.npmjs.com/package/npm);
-* run `npm install` inside `./web`;
+* run `npm install` inside `./frontend`;
 * run `npm run build` there;
-* result files will be saved in `./web/public`.
+* result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
 It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
@@ -448,6 +493,10 @@ npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
+#### Frontend guide
+
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
 ## API
 
 ### Authorization
@@ -481,8 +530,10 @@ type Comment struct {
     User      User            `json:"user"`    // user info, read only
     Locator   Locator         `json:"locator"` // post locator
     Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
+    Vote      int             `json:"vote"`    // vote for the current user, -1/1/0.
+    Controversy float64       `json:"controversy,omitempty"` // comment controversy, read only
     Timestamp time.Time       `json:"time"`    // time stamp, read only
+    Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
     PostTitle string          `json:"title"`   // post title
@@ -492,6 +543,11 @@ type Locator struct {
     SiteID string `json:"site"`     // site id
     URL    string `json:"url"`      // post url
 }
+
+type Edit struct {
+  Timestamp time.Time `json:"time" bson:"time"`
+  Summary   string    `json:"summary"`
+}
 ```
 
 * `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
@@ -518,11 +574,11 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
 ```go
- 	type EditRequest struct {
- 		Text    string `json:"text"`    // updated text
- 		Summary string `json:"summary"` // optional, summary of the edit
- 		Delete  bool   `json:"delete"`  // delete flag 
- 	}{}
+   type EditRequest struct {
+     Text    string `json:"text"`    // updated text
+     Summary string `json:"summary"` // optional, summary of the edit
+     Delete  bool   `json:"delete"`  // delete flag
+   }{}
 ```
 
 * `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
@@ -553,23 +609,36 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go
-  type config struct {
-      Version       string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
-      Admins        []string `json:"admins"`
-      Auth          []string `json:"auth_providers"`
-      LowScore      int      `json:"low_score"`
-      CriticalScore int      `json:"critical_score"`
+  type Config struct {
+      Version        string   `json:"version"`
+      EditDuration   int      `json:"edit_duration"`
+      MaxCommentSize int      `json:"max_comment_size"`
+      Admins         []string `json:"admins"`
+      AdminEmail     string   `json:"admin_email"`
+      Auth           []string `json:"auth_providers"`
+      LowScore       int      `json:"low_score"`
+      CriticalScore  int      `json:"critical_score"`
+      PositiveScore  bool     `json:"positive_score"`
+      ReadOnlyAge    int      `json:"readonly_age"`
+      MaxImageSize   int      `json:"max_image_size"`
   }
-  ``` 
+  ```
+
 * `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
-  
+
 ### RSS feeds
-  
+
 * `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -595,7 +664,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
 _all admin calls require auth and admin privilege_
 
-## Privacy 
+## Privacy
 
 * Remark42 is trying to be very sensitive to any private or semi-private information.
 * Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
@@ -606,7 +675,7 @@ _all admin calls require auth and admin privilege_
 * There are no third-party analytic services involved.
 * User can request all information remark42 knows about and export to gz file.
 * Supported complete cleanup of all information related to user's activity.
-* Cookie lifespan can be restricted to session-only. 
+* Cookie lifespan can be restricted to session-only.
 * All potentially sensitive data stored by remark42 hashed and encrypted.
 
 ## Technical details

backend/Gopkg.lock

@@ -1,429 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter",
-  ]
-  pruneopts = "UT"
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
-  name = "github.com/globalsign/mgo"
-  packages = [
-    ".",
-    "bson",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = "UT"
-  revision = "113d3961e7311526535a1ef7042196563d442761"
-  version = "r2018.06.15"
-
-[[projects]]
-  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware",
-  ]
-  pruneopts = "UT"
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
-  name = "github.com/go-chi/cors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dba6525398619dead495962a916728e7ee2ca322"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:38940db39a8ee330d3e7f68ab4c298575462f5986ae2bb0cafa86f095e9d32ad"
-  name = "github.com/go-pkgz/auth"
-  packages = [
-    ".",
-    "avatar",
-    "logger",
-    "middleware",
-    "provider",
-    "token",
-  ]
-  pruneopts = "UT"
-  revision = "7597c083287c33ba5362687a7815a032fd92c418"
-  version = "v0.4.1"
-
-[[projects]]
-  digest = "1:7b1f422f560b103f435f8501f854f238f1ce00d0f8146e76eacdfa71f2c8d8c0"
-  name = "github.com/go-pkgz/lgr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dbd940fec9650260d15aa071cab658d0502ed92f"
-  version = "v0.3.2"
-
-[[projects]]
-  digest = "1:c509e3f646c48148f2239fa66d5cd62785dfd057b5d2b46972e45b60049c19ed"
-  name = "github.com/go-pkgz/mongo"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ee1e9a167094f8cdbc27407907aa694fff350ec3"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
-  name = "github.com/go-pkgz/repeater"
-  packages = [
-    ".",
-    "strategy",
-  ]
-  pruneopts = "UT"
-  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:9aba5c95373481f118e57e9740d9e82f86802d86849809ab89324bcb4f236451"
-  name = "github.com/go-pkgz/rest"
-  packages = [
-    ".",
-    "cache",
-    "logger",
-  ]
-  pruneopts = "UT"
-  revision = "e7d08d0194d613b8854de2e487bf7732500fa153"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = "UT"
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
-  name = "github.com/nullrocks/identicon"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7875f45b0022edded6377e40639d8aa620193a62"
-
-[[projects]]
-  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  pruneopts = "UT"
-  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
-  version = "v0.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
-  name = "golang.org/x/crypto"
-  packages = [
-    "acme",
-    "acme/autocert",
-  ]
-  pruneopts = "UT"
-  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64",
-  ]
-  pruneopts = "UT"
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom",
-  ]
-  pruneopts = "UT"
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex",
-  ]
-  pruneopts = "UT"
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = "UT"
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = "UT"
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/PuerkitoBio/goquery",
-    "github.com/coreos/bbolt",
-    "github.com/dgrijalva/jwt-go",
-    "github.com/didip/tollbooth",
-    "github.com/didip/tollbooth_chi",
-    "github.com/globalsign/mgo",
-    "github.com/globalsign/mgo/bson",
-    "github.com/go-chi/chi",
-    "github.com/go-chi/chi/middleware",
-    "github.com/go-chi/cors",
-    "github.com/go-chi/render",
-    "github.com/go-pkgz/auth",
-    "github.com/go-pkgz/auth/avatar",
-    "github.com/go-pkgz/auth/provider",
-    "github.com/go-pkgz/auth/token",
-    "github.com/go-pkgz/lgr",
-    "github.com/go-pkgz/mongo",
-    "github.com/go-pkgz/repeater",
-    "github.com/go-pkgz/rest",
-    "github.com/go-pkgz/rest/cache",
-    "github.com/go-pkgz/rest/logger",
-    "github.com/google/uuid",
-    "github.com/gorilla/feeds",
-    "github.com/hashicorp/go-multierror",
-    "github.com/jessevdk/go-flags",
-    "github.com/microcosm-cc/bluemonday",
-    "github.com/patrickmn/go-cache",
-    "github.com/pkg/errors",
-    "github.com/rakyll/statik/fs",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/acme/autocert",
-    "golang.org/x/net/html",
-    "gopkg.in/russross/blackfriday.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/Gopkg.toml

@@ -1,44 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-
-[[constraint]]
-  name = "github.com/globalsign/mgo"
-  version = "r2018.06.15"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-
-
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"

backend/app/cmd/avatar.go

@@ -31,6 +31,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL)
+	r, err := http.Get(listURL) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL)
+		r, err = http.Get(commentsURL) // nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}
@@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil

backend/app/cmd/cmd.go

@@ -33,7 +33,7 @@ type CommonOpts struct {
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
-	c.RemarkURL = commonOpts.RemarkURL
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
 	c.SharedSecret = commonOpts.SharedSecret
 	c.Revision = commonOpts.Revision
 }

backend/app/cmd/server.go

@@ -8,13 +8,14 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	auth_cache "github.com/patrickmn/go-cache"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
@@ -31,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -42,6 +44,7 @@ type ServerCommand struct {
 	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -53,23 +56,25 @@ type ServerCommand struct {
 	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
 	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
 	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
 	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
 	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
 	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
 	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" default:"" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
 
 	Auth struct {
 		TTL struct {
 			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
 			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
 		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Dev      bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
@@ -90,6 +95,22 @@ type StoreGroup struct {
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
 }
 
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+}
+
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
@@ -159,6 +180,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -179,6 +201,7 @@ func (s *ServerCommand) Execute(args []string) error {
 	app, err := s.newServerApp()
 	if err != nil {
 		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
 		log.Printf("[ERROR] remark terminated with error %+v", err)
@@ -211,12 +234,20 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
+
 	dataService := &service.DataStore{
 		Interface:              storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
+		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
@@ -272,15 +303,16 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     imageService,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+		da, errDevAuth := authenticator.DevAuth()
+		if errDevAuth != nil {
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -294,6 +326,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -319,12 +352,17 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -401,6 +439,28 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+				MaxHeight:  s.Image.ResizeHeight,
+				MaxWidth:   s.Image.ResizeWidth,
+			},
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      s.EditDuration + time.Second, // add extra second to image TTL for staging
+		}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
@@ -471,10 +531,29 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 	if s.Auth.Dev {
+		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Anonymous {
+		log.Print("[INFO] anonymous access enabled")
+		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
+		authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
+			user = strings.TrimSpace(user)
+			if len(user) < 3 {
+				log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
+				return false, nil
+			}
+
+			if !isValidAnonName(user) {
+				log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
+				return false, nil
+			}
+			return true, nil
+		}))
+	}
+
 	if providers == 0 {
 		log.Printf("[WARN] no auth providers defined")
 	}
@@ -551,6 +630,7 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
@@ -563,17 +643,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*auth_cache.Cache
+	*authcache.Cache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
 }
 
+// Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 	return c.Cache.Get(key.(string))
 }
 
+// Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
 }

backend/app/cmd/server_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
@@ -12,20 +13,21 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/mongo"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := rand.Intn(40000) + 10000
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
@@ -33,7 +35,7 @@ func TestServerApp(t *testing.T) {
 	time.Sleep(100 * time.Millisecond) // let server start
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -43,7 +45,7 @@ func TestServerApp(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
@@ -59,8 +61,9 @@ func TestServerApp(t *testing.T) {
 }
 
 func TestServerApp_DevMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
@@ -72,7 +75,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -83,6 +86,49 @@ func TestServerApp_DevMode(t *testing.T) {
 	app.Wait()
 }
 
+func TestServerApp_AnonMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = port
+		o.Auth.Anonymous = true
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+
+	// send ping
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	// try to login with good name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+
+	// try to login with bad name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	// try to login with short name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	app.Wait()
+}
 func TestServerApp_WithMongo(t *testing.T) {
 
 	mongoURL := os.Getenv("MONGO_TEST")
@@ -102,15 +148,15 @@ func TestServerApp_WithMongo(t *testing.T) {
 		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
 	require.Nil(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation = "/tmp"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
 
 	// create app
 	app, err := opts.newServerApp()
 	require.Nil(t, err)
 
 	defer func() {
-		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, err)
+		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
+		assert.NoError(t, e)
 		conn := mongo.NewConnection(s, "test_remark", "")
 		_ = conn.WithDB(func(dbase *mgo.Database) error {
 			assert.NoError(t, dbase.DropDatabase())
@@ -145,8 +191,10 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
-		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
+		"--ssl.port=18443", "--image.fs.path=/tmp"})
 	require.Nil(t, err)
 
 	// create app
@@ -200,7 +248,7 @@ func TestServerApp_Failed(t *testing.T) {
 	p := flags.NewParser(&opts, flags.Default)
 
 	// RO bolt location
-	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
 	assert.Nil(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
@@ -240,7 +288,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 func TestServerApp_Shutdown(t *testing.T) {
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+		o.Port = rand.Intn(40000) + 10000
 		return o
 	})
 	st := time.Now()
@@ -264,7 +312,7 @@ func TestServerApp_MainSignal(t *testing.T) {
 
 	p := flags.NewParser(&s, flags.Default)
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
 	require.Nil(t, err)
@@ -316,8 +364,9 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+	port := rand.Intn(40000) + 10000
+	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
@@ -345,8 +394,8 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	client := http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
 	require.Nil(t, err)
@@ -356,7 +405,8 @@ func TestServerAuthHooks(t *testing.T) {
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
+	req, e := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
 	assert.Nil(t, e)
 	req.SetBasicAuth("admin", "password")
 	resp, e = client.Do(req)
@@ -370,7 +420,7 @@ func TestServerAuthHooks(t *testing.T) {
 	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
 
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
 	require.Nil(t, err)
@@ -390,7 +440,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
 	require.Nil(t, err)
-	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
@@ -415,5 +465,6 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 		log.Print("[TEST] terminate app")
 		cancel()
 	}()
+	rand.Seed(time.Now().UnixNano())
 	return app, ctx
 }

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )
@@ -62,10 +62,10 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 		return
 	}
-	log.Setup(log.Msec, log.LevelBraces, log.CallerPkg, log.CallerIgnore("logger"))
+	log.Setup(log.Msec, log.LevelBraces)
 }
 
 // getDump reads runtime stack and returns as a string

backend/app/main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -11,20 +12,25 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(500 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		time.Sleep(5000 * time.Millisecond)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.Nil(t, e)
 	}()
 
 	wg := sync.WaitGroup{}
@@ -32,20 +38,29 @@ func TestMain(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
 		wg.Done()
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	var passed bool
+	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
+		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		if e != nil {
+			t.Logf("%+v", e)
+			return e
+		}
+		require.Nil(t, e)
+		defer resp.Body.Close()
+		assert.Equal(t, 200, resp.StatusCode)
+		body, e := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, e)
+		assert.Equal(t, "pong", string(body))
+		passed = true
+		return nil
+	})
 
-	// send ping
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
+	assert.NoError(t, err)
+	assert.Equal(t, true, passed, "at least on ping passed")
 
 	wg.Wait()
 }

backend/app/migrator/backup_test.go

@@ -16,7 +16,8 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {

backend/app/migrator/disqus.go

@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue

backend/app/migrator/native.go

@@ -2,23 +2,28 @@ package migrator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
+	"sync/atomic"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/syncs"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
+const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
 // {"version": 1, comments:[{...}\n,{}], meta: {meta}}
 // each comments starts from the new line
 type Native struct {
-	DataStore Store
+	DataStore  Store
+	Concurrent int
 }
 
 type meta struct {
@@ -45,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
 		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -70,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -91,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -99,8 +104,13 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, err
 	}
 
-	failed := 0
-	total, comments := 0, 0
+	var failed, total, comments int64
+
+	concurrent := defaultConcurrent
+	if n.Concurrent > 0 {
+		concurrent = n.Concurrent
+	}
+	grp := syncs.NewSizedGroup(concurrent, syncs.Preemptive)
 
 	for {
 		comment := store.Comment{}
@@ -112,27 +122,34 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		total++
 
 		if err != nil {
+			atomic.AddInt64(&failed, 1)
 			failed++
 			continue
 		}
 
-		if _, err = n.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
+		// write comments in parallel
+		grp.Go(func(context.Context) {
+			if _, e := n.DataStore.Create(comment); e != nil {
+				atomic.AddInt64(&failed, 1)
+				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
+				return
+			}
+			num := atomic.AddInt64(&comments, 1)
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
+			}
+		})
+
 	}
 
+	grp.Wait()
+
 	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
+		return int(comments), errors.Errorf("failed to save %d comments", failed)
 	}
 	log.Printf("[INFO] imported %d comments from %d records", comments, total)
 
 	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
 
-	return comments, err
+	return int(comments), err
 }

backend/app/migrator/wordpress.go

@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue
@@ -142,7 +143,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -17,7 +17,7 @@ import (
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
 	assert.Nil(t, err, "create store")
 

backend/app/notify/notify.go

@@ -29,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment

backend/app/notify/telegram.go

@@ -42,7 +42,10 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	}
 	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
-	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	err := repeater.NewDefault(5, time.Millisecond*250).Do(ctx, func() error {
 		client := http.Client{Timeout: telegramTimeOut}
 		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
 		if err != nil {
@@ -110,10 +113,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/notify/telegram_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NotNil(t, err, "should failed")
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())

backend/app/rest/api/admin.go

@@ -27,25 +27,6 @@ type admin struct {
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
-}
-
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
 func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
@@ -55,7 +36,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	err := a.dataService.Delete(locator, id, store.SoftDelete)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
@@ -71,7 +52,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
 	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -88,7 +69,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 
 	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
 	if err != nil || len(ucomments) == 0 {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
 	}
 	render.Status(r, http.StatusOK)
@@ -103,7 +84,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token", rest.ErrActionRejected)
 		return
 	}
 
@@ -111,19 +92,19 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
 	if !claims.User.BoolAttr("delete_me") {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token", rest.ErrNoAccess)
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+		avatarStore := a.authenticator.AvatarProxy().Store
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
 	}
@@ -147,7 +128,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -159,7 +140,7 @@ func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	users, err := a.dataService.Blocked(siteID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
 	}
 	render.JSON(w, r, users)
@@ -178,13 +159,14 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
 	if !roStatus {
 		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
+			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+				"read-only due the age", rest.ErrActionRejected)
 			return
 		}
 	}
 
 	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status", rest.ErrPostNotFound)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
@@ -198,7 +180,7 @@ func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
 
 	c, err := a.dataService.SetTitle(locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title", rest.ErrInternal)
 		return
 	}
 	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
@@ -215,7 +197,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	verifyStatus := r.URL.Query().Get("verified") == "1"
 
 	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
@@ -230,7 +212,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	pinStatus := r.URL.Query().Get("pin") == "1"
 
 	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))

backend/app/rest/api/admin_test.go

@@ -58,12 +58,11 @@ func TestAdmin_Delete(t *testing.T) {
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 
 	// delete a comment
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	requireAdminOnly(t, req)
+	resp, err = sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -75,6 +74,7 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
 
+	time.Sleep(250 * time.Millisecond)
 	// check last comments updated
 	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
 	assert.Equal(t, 200, code)
@@ -112,11 +112,13 @@ func TestAdmin_Title(t *testing.T) {
 	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -131,13 +133,12 @@ func TestAdmin_Title(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
@@ -167,38 +168,37 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	_, err = srv.DataService.Create(c3)
 	assert.NoError(t, err)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
+	cmntWithInfo := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &cmntWithInfo)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
+	assert.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
 
 	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
+	assert.Equal(t, id1, cmntWithInfo.Comments[0].ID)
+	assert.Equal(t, "o test test #1", cmntWithInfo.Comments[0].Orig)
+	assert.False(t, cmntWithInfo.Comments[0].Deleted)
+	t.Logf("%+v", cmntWithInfo.Comments[0].User)
 
 	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Orig)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
+	assert.True(t, cmntWithInfo.Comments[1].Deleted)
 
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Orig)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
+	assert.True(t, cmntWithInfo.Comments[2].Deleted)
 }
 
 func TestAdmin_Pin(t *testing.T) {
@@ -218,6 +218,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
+		requireAdminOnly(t, req)
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
@@ -259,15 +260,14 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Nil(t, err)
 
 	block := func(val int, ttl string) (code int, body []byte) {
-		client := http.Client{}
 		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
 		if ttl != "" {
 			url = url + "&ttl=" + ttl
 		}
 		req, e := http.NewRequest(http.MethodPut, url, nil)
 		assert.Nil(t, e)
-		req.SetBasicAuth("admin", "password")
-		resp, e := client.Do(req)
+		requireAdminOnly(t, req)
+		resp, e := sendReq(t, req, adminUmputunToken)
 		require.Nil(t, e)
 		body, e = ioutil.ReadAll(resp.Body)
 		assert.Nil(t, e)
@@ -329,29 +329,26 @@ func TestAdmin_BlockedList(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
-
 	// block user1
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	res, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	res, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
@@ -363,10 +360,9 @@ func TestAdmin_BlockedList(t *testing.T) {
 	time.Sleep(50 * time.Millisecond)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	res, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
@@ -393,15 +389,15 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	resp, err := sendReq(t, req, "") // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
@@ -413,20 +409,18 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err := json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
@@ -437,10 +431,9 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err = json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
@@ -448,15 +441,13 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.NotNil(t, err)
@@ -485,30 +476,27 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, info.ReadOnly, "ro by age")
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 }
@@ -529,13 +517,13 @@ func TestAdmin_Verify(t *testing.T) {
 	verified := srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.True(t, verified)
 
@@ -551,9 +539,9 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
@@ -598,12 +586,11 @@ func TestAdmin_ExportFile(t *testing.T) {
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 
 	assert.Equal(t, 200, resp.StatusCode)
 	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
@@ -662,9 +649,10 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
+
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	_, err = srv.DataService.User("radio-t", "user1", 0, 0)

backend/app/rest/api/migrator.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -40,15 +39,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
@@ -56,13 +46,14 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	tmpfile, err := m.saveTemp(r.Body)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -78,25 +69,26 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	if err := r.ParseMultipartForm(20 * 1024 * 1024); err != nil { // 20M max memory, if bigger will make a file
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
 		return
 	}
 
 	file, _, err := r.FormFile("file")
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import from the request")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
 	tmpfile, err := m.saveTemp(file)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -155,7 +147,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
 		return
 	}
 }

backend/app/rest/api/migrator_test.go

@@ -3,43 +3,29 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/store"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -50,19 +36,12 @@ func TestMigrator_Import(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -77,7 +56,7 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=radio-t&provider=native", contentType, bodyBuf)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
@@ -85,23 +64,17 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 }
+
 func TestMigrator_ImportFromWP(t *testing.T) {
-	srv, ds, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress", r)
 	assert.Nil(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
@@ -113,40 +86,18 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.NoError(t, ds.Interface.Close())
-
-	tsAccess, _, teardownAccess := startupT(t)
-	defer teardownAccess()
-
-	res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
-	require.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-	t.Logf("%+v", comments)
-	assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
-		comments[0].Text)
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
@@ -154,18 +105,17 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -173,28 +123,27 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
 	resp, err := client.Do(req)
@@ -202,40 +151,35 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t&timeout=100ms", nil)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
+	ts, _, teardown := startupT(t)
+	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
+	waitForImportCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -252,7 +196,7 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -266,42 +210,25 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t", nil)
 	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc = &Migrator{
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
-		Cache:             &cache.Nop{},
-		KeyStore:          adminStore,
-	}
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	return svc, dataStore, ts
-}
-
-func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
-	ts.Close()
-	_ = os.Remove(testDb)
+func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -5,9 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
 	"strings"
 	"sync"
 	"time"
@@ -17,7 +15,6 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/cors"
-	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -30,6 +27,7 @@ import (
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -44,6 +42,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     *image.Service
 
 	WebRoot         string
 	RemarkURL       string
@@ -80,6 +79,7 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		err := s.httpServer.ListenAndServe()
@@ -89,7 +89,10 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		go func() {
@@ -106,7 +109,11 @@ func (s *Rest) Run(port int) {
 		m := s.makeAutocertManager()
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.lock.Unlock()
 
 		go func() {
@@ -179,17 +186,17 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Flags(logger.All), logger.Log(log.Default()), logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
-		r.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
 
@@ -199,16 +206,15 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
-			rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
 		})
 
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/find", s.findCommentsCtrl)
 			ropen.Get("/id/{id}", s.commentByIDCtrl)
 			ropen.Get("/comments", s.findUserCommentsCtrl)
@@ -219,82 +225,145 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
+			ropen.Get("/img", s.ImageProxy.Handler)
 
-			ropen.Mount("/rss", s.rssRoutes())
-			ropen.Mount("/img", s.ImageProxy.Routes())
+			ropen.Route("/rss", func(rrss chi.Router) {
+				rrss.Get("/post", s.rssPostCommentsCtrl)
+				rrss.Get("/site", s.rssSiteCommentsCtrl)
+				rrss.Get("/reply", s.rssRepliesCtrl)
+			})
+		})
+
+		// open routes, cached
+		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
+			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(authMiddleware.Auth, middleware.NoCache, logInfoWithBody)
 			rauth.Get("/user", s.userInfoCtrl)
 			rauth.Get("/userdata", s.userAllDataCtrl)
+		})
 
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		// admin routes, require auth and admin users only
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminService.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminService.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminService.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminService.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminService.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminService.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminService.setPinCtrl)
+			radmin.Get("/blocked", s.adminService.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminService.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminService.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminService.migrator.exportCtrl)
+			radmin.Post("/import", s.adminService.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminService.migrator.importFormCtrl)
+			radmin.Get("/import/wait", s.adminService.migrator.importWaitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
 			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(middleware.NoCache)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
 			rauth.Put("/comment/{id}", s.updateCommentCtrl)
 			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
 		})
+
+		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, rejectAnonUser)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Post("/picture", s.savePictureCtrl)
+		})
+
 	})
 
-	// respond to /robots.txt with the list of allowed paths
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
+	// open routes on root level
+	router.Group(func(rroot chi.Router) {
+		tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))
+		rroot.Get("/index.html", s.getStartedCtrl)
+		rroot.Get("/robots.txt", s.getRobotsCtrl)
+	})
 
 	// file server for static content from /web
 	addFileServer(router, "/web", http.Dir(s.WebRoot))
 	return router
 }
 
+func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+
+	res = s.adminService.alterComments(comments, r) // apply admin's alteration
+
+	// prepare vote info for client view
+	vote := func(c store.Comment, r *http.Request) store.Comment {
+
+		c.Vote = 0 // default is "none" (not voted)
+
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			c.Votes = nil // hide voters list and don't set Vote for non-authed user
+			return c
+		}
+
+		if v, ok := c.Votes[user.ID]; ok {
+			if v {
+				c.Vote = 1
+			} else {
+				c.Vote = -1
+			}
+		}
+
+		c.Votes = nil // hide voters list
+		return c
+	}
+
+	for i, c := range res {
+		c = vote(c, r)
+		res[i] = c
+	}
+
+	return res
+}
+
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
 // serves static files from /web or embedded by statik
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 
 	var webFS http.Handler
 
 	statikFS, err := fs.New()
-	if err == nil {
-		log.Printf("[INFO] run file server for %s, embedded", root)
-		webFS = http.FileServer(statikFS)
-	}
 	if err != nil {
 		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
 		log.Printf("[INFO] run file server for %s, path %s", root, path)
 		webFS = http.FileServer(root)
+	} else {
+		log.Printf("[INFO] run file server for %s, embedded", root)
+		webFS = http.FileServer(statikFS)
 	}
 
 	origPath := path
@@ -340,9 +409,42 @@ func filterComments(comments []store.Comment, fn func(c store.Comment) bool) []s
 // admins will have different keys in order to prevent leak of admin-only data to regular users
 func URLKey(r *http.Request) string {
 	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
-		key = adminPrefix + key
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil && user.Admin {
+		key = adminPrefix + key // make separate cache key for admins
 	}
 	return key
 }
+
+// URLKeyWithUser gets url from request to use it as cache key and attaching user ID
+// admins will have different keys in order to prevent leak of admin-only data to regular users
+func URLKeyWithUser(r *http.Request) string {
+	adminPrefix := "admin!!"
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil {
+		if user.Admin {
+			key = adminPrefix + user.ID + "!!" + key // make separate cache key for admins
+		} else {
+			key = user.ID + "!!" + key // make separate cache key for authed users
+		}
+	}
+	return key
+}
+
+// rejectAnonUser is a middleware rejecting anonymous users
+func rejectAnonUser(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		if strings.HasPrefix(user.ID, "anonymous_") {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}

backend/app/rest/api/rest_private.go

@@ -9,14 +9,14 @@ import (
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
@@ -28,7 +28,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -40,36 +40,36 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment.Orig = comment.Text // original comment text, prior to md render
 	if err := s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	comment = s.CommentFormatter.Format(comment)
 
 	// check if user blocked
 	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	if s.isReadOnly(comment.Locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	id, err := s.DataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment", rest.ErrInternal)
 		return
 	}
 
 	// DataService modifies comment
 	finalComment, err := s.DataService.Get(comment.Locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
@@ -95,7 +95,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -108,12 +108,13 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	var currComment store.Comment
 	var err error
 	if currComment, err = s.DataService.Get(locator, id); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
 
 	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+			"can not edit comments for other users", rest.ErrNoAccess)
 		return
 	}
 
@@ -126,11 +127,13 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	res, err := s.DataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
+
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
+		code := s.parseError(err, rest.ErrCommentRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
 		return
 	}
 
@@ -158,19 +161,20 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	vote := r.URL.Query().Get("vote") == "1"
 
 	if s.isReadOnly(locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	// check if user blocked
 	if s.adminService.checkBlocked(locator.SiteID, user) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
+		code := s.parseError(err, rest.ErrVoteRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
@@ -183,7 +187,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info", rest.ErrInternal)
 		return
 	}
 
@@ -209,14 +213,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
+		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
+		b, errUser := json.Marshal(comments)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
@@ -228,7 +232,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	merr = multierror.Append(merr, write([]byte(`}`)))
 	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info", rest.ErrInternal)
 		return
 	}
 
@@ -258,7 +262,7 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 
 	tokenStr, err := s.Authenticator.TokenService().Token(claims)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
@@ -266,6 +270,31 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
+// POST /image - save image with form request
+func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"id": id})
+}
+
 func (s *Rest) isReadOnly(locator store.Locator) bool {
 	if s.ReadOnlyAge > 0 {
 		// check RO by age
@@ -275,3 +304,28 @@ func (s *Rest) isReadOnly(locator store.Locator) bool {
 	}
 	return s.DataService.IsReadOnly(locator) // ro manually
 }
+
+func (s *Rest) parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private_test.go

@@ -1,22 +1,37 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/rest"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
 func TestRest_Create(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -289,7 +304,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	body, err := ioutil.ReadAll(b.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
+	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
 	client = http.Client{}
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
@@ -343,7 +358,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("admin", "password")
+		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -351,22 +366,65 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, 200, vote(1), "first vote allowed")
 	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"admin": true}, cr.Votes)
+	assert.Equal(t, 1, cr.Vote)
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
+	assert.Equal(t, 0, cr.Vote)
+
+	assert.Equal(t, 200, vote(-1), "opposite vote allowed one more time")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	assert.Equal(t, 400, vote(-1), "dbl vote not allowed")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	req, err := http.NewRequest("GET",
+		fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+	assert.NoError(t, err)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	cr = store.Comment{}
+	err = json.NewDecoder(resp.Body).Decode(&cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -432,13 +490,12 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
 
-	for i := 0; i < 478; i++ {
+	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
 		c.Timestamp = c.Timestamp.Add(time.Second)
 		_, err := srv.DataService.Create(c)
 		require.Nil(t, err)
 	}
-
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
@@ -454,7 +511,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
 		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
+	assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
@@ -489,3 +546,163 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	savePic := func(name string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 1462, len(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+}
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	svc.ImageService = &image.Service{
+		Store: &image.FileSystem{
+			Staging:  "/tmp/remark42/images.staging",
+			Location: "/tmp/remark42/images",
+			MaxSize:  2000,
+		},
+		TTL: time.Millisecond * 100,
+	}
+	svc.DataService.EditDuration = time.Millisecond * 100
+	svc.DataService.ImageService = svc.ImageService
+
+	uploadPicture := func(file string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.Contains(t, m["id"], ".png")
+		t.Logf(string(body))
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png")
+	id2 := uploadPicture("pic2.png")
+	id3 := uploadPicture("pic3.png")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	svc := Rest{}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := svc.parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}

backend/app/rest/api/rest_public.go

@@ -1,9 +1,12 @@
 package api
 
 import (
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 
@@ -15,9 +18,10 @@ import (
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
 // find comments for given post. Returns in tree or plain formats, sorted
 func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -27,19 +31,19 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
 
-	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Find(locator, sort)
 		if e != nil {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.adminService.alterComments(comments, r)
+		maskedComments := s.alterComments(comments, r)
 		var b []byte
 		switch r.URL.Query().Get("format") {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(maskedComments, sort, s.ReadOnlyAge)
 			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*rest.Node{}
+				tree.Nodes = []*service.Node{}
 			}
 			if s.DataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
@@ -56,7 +60,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -69,19 +73,19 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
 	user, err := rest.GetUserInfo(r)
 	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info", rest.ErrNoAccess)
 		return
 	}
 	comment.User = user
 	comment.Orig = comment.Text
 	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
@@ -104,7 +108,7 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info", rest.ErrPostNotFound)
 		return
 	}
 
@@ -129,14 +133,14 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments", rest.ErrInternal)
 		return
 	}
 
@@ -156,10 +160,10 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
+	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -185,13 +189,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
-	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID)
+	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.User(siteID, userID, limit, 0)
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		count, e := s.DataService.UserCount(siteID, userID)
 		if e != nil {
@@ -202,7 +206,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -224,7 +228,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		Auth           []string `json:"auth_providers"`
 		LowScore       int      `json:"low_score"`
 		CriticalScore  int      `json:"critical_score"`
+		PositiveScore  bool     `json:"positive_score"`
 		ReadOnlyAge    int      `json:"readonly_age"`
+		MaxImageSize   int      `json:"max_image_size"`
 	}
 
 	cnf := config{
@@ -235,7 +241,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		AdminEmail:     s.DataService.AdminStore.Email(siteID),
 		LowScore:       s.ScoreThresholds.Low,
 		CriticalScore:  s.ScoreThresholds.Critical,
+		PositiveScore:  s.DataService.PositiveScore,
 		ReadOnlyAge:    s.ReadOnlyAge,
+		MaxImageSize:   s.ImageService.Store.SizeLimit(),
 	}
 
 	cnf.Auth = []string{}
@@ -255,29 +263,26 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	count, err := s.DataService.Count(locator)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
 	}
 	render.JSON(w, r, R.JSON{"count": count, "locator": locator})
 }
 
-// POST /count?site=siteID - get number of comments for posts from post body
+// POST /counts?site=siteID - get number of comments for posts from post body
 func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
 		return
 	}
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
-	if _, err := hasher.Write([]byte(k)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	h := sha1.Sum([]byte(k)) // nolint
+	sha := base64.URLEncoding.EncodeToString(h[:])
+
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
@@ -288,7 +293,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -320,7 +325,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -328,3 +333,70 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't render posts lits for site %s", siteID)
 	}
 }
+
+// GET /picture/{user}/{id} - get picture
+func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
+	imgRdr, size, err := s.ImageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer func() {
+		if e := imgRdr.Close(); e != nil {
+			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
+		}
+	}()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *Rest) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *Rest) getRobotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}

backend/app/rest/api/rest_public_test.go

@@ -14,8 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -111,7 +111,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -137,7 +137,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
@@ -178,7 +178,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -186,7 +186,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
+	tree = service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -377,6 +377,37 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 3, pi[1].Count)
 }
 
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
+}
+
 func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -392,7 +423,9 @@ func TestRest_Config(t *testing.T) {
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
 	assert.Equal(t, -10., j["critical_score"])
+	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
+	assert.Equal(t, 10000., j["max_image_size"])
 	t.Logf("%+v", j)
 }
 
@@ -441,5 +474,5 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }

backend/app/rest/api/rest_test.go

@@ -4,10 +4,13 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -22,19 +25,22 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
 var testHTML = "/tmp/test-remark.html"
 var getStartedHTML = "/tmp/getstarted.html"
 
 var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.gAR_sZT7hTx7CNHByyrJQWMB5tAtoiISAiG8kes1IjA"
+
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -175,7 +181,82 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 	srv.Shutdown()
 }
 
+func TestRest_rejectAnonUser(t *testing.T) {
+
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello")
+	}))))
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
+
+	resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
+
+	resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
+}
+
+func Test_URLKey(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKey(r))
+		})
+	}
+
+}
+
+func Test_URLKeyWithUser(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "user!!http://example.com/1"},
+		{"http://example.com/2", store.User{ID: "user2"}, "user2!!http://example.com/2"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!user!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKeyWithUser(r))
+		})
+	}
+
+}
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+
+	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
+	os.Remove(testDb)
+	os.Remove(testHTML)
+	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
+
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 
@@ -204,7 +285,15 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+				Staging:    "/tmp/pics-remark42/staging",
+			},
+			TTL: time.Millisecond * 100,
+		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -226,15 +315,30 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 
 	teardown = func() {
 		ts.Close()
-		srv.DataService.Close()
+		require.NoError(t, srv.DataService.Close())
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
+// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+func fakeAuth(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("fake_id") != "" {
+			r = rest.SetUserInfo(r, store.User{
+				ID:   r.URL.Query().Get("fake_id"),
+				Name: r.URL.Query().Get("fake_name"),
+			})
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
 	require.Nil(t, err)
@@ -244,6 +348,14 @@ func get(t *testing.T, url string) (string, int) {
 	return string(body), r.StatusCode
 }
 
+func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+	if token != "" {
+		r.Header.Set("X-JWT", token)
+	}
+	return client.Do(r)
+}
+
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
@@ -297,3 +409,13 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}

backend/app/rest/api/rss.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
@@ -16,20 +15,12 @@ import (
 )
 
 const maxRssItems = 20
-const maxLastCommentsReply = 1000
-const maxReplyDuration = 24 * time.Hour
+const maxLastCommentsReply = 5000
+const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
 func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -41,8 +32,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments)
+		comments = s.alterComments(comments, r)
+		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
@@ -50,14 +41,14 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrPostNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -73,9 +64,9 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
+		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
@@ -83,13 +74,13 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -100,18 +91,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
+	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
 	data, err := s.Cache.Get(key, func() (res []byte, e error) {
 		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		replies := []store.Comment{}
 		for _, c := range comments {
 			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
 				break
 			}
+			if c.User.ID != userID {
+				userName = c.User.Name
+			}
 			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
 				var pc store.Comment
 				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
@@ -123,7 +118,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		rss, e := s.toRssFeed(siteID, replies)
+		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
@@ -131,19 +126,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
+func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
+	if description == "" {
+		description = "comment updates"
+	}
 	lastCommentTS := time.Unix(0, 0)
 	if len(comments) > 0 {
 		lastCommentTS = comments[0].Timestamp
@@ -152,7 +150,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 	feed := &feeds.Feed{
 		Title:       "Remark42 comments",
 		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
+		Description: description,
 		Created:     lastCommentTS,
 	}
 

backend/app/rest/api/rss_test.go

@@ -38,7 +38,7 @@ func TestServer_RssPost(t *testing.T) {
         <channel>
             <title>Remark42 comments</title>
             <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
+            <description>post comments for https://radio-t.com/blah1</description>
             <pubDate>%s</pubDate>
             <item>
 		      <title>developer one</title>
@@ -93,7 +93,7 @@ func TestServer_RssSite(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>radio-t</link>
-		    <description>comment updates</description>
+		    <description>site comment for radio-t</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -156,7 +156,7 @@ func TestServer_RssWithReply(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
+		    <description>post comments for https://radio-t.com/blah10</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one &gt; developer one</title>
@@ -242,7 +242,7 @@ func TestServer_RssReplies(t *testing.T) {
 	    <channel>
 	        <title>Remark42 comments</title>
 	        <link>radio-t</link>
-	        <description>comment updates</description>
+	        <description>replies to user2</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>

backend/app/rest/httperrors.go

@@ -9,16 +9,40 @@ import (
 
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest"
+)
+
+// All error codes for UI mapping and translation
+const (
+	ErrInternal           = 0  // any internal error
+	ErrCommentNotFound    = 1  // can't find comment
+	ErrDecode             = 2  // failed to unmarshal incoming request
+	ErrNoAccess           = 3  // rejected by auth
+	ErrCommentValidation  = 4  // validation failed
+	ErrPostNotFound       = 5  // can't find post
+	ErrSiteNotFound       = 6  // can't find site
+	ErrUserBlocked        = 7  // user blocked
+	ErrReadOnly           = 8  // write failed on read only
+	ErrCommentRejected    = 9  // general error on rejected comment change
+	ErrCommentEditExpired = 10 // too late for edit
+	ErrCommentEditChanged = 11 // parent comment cannot be changed
+	ErrVoteRejected       = 12 // general error on vote rejected
+	ErrVoteSelf           = 13 // vote for own comment
+	ErrVoteDbl            = 14 // already voted for the comment
+	ErrVoteMax            = 15 // too many votes for the comment
+	ErrVoteMinScore       = 16 // min score reached for the comment
+	ErrActionRejected     = 17 // general error for rejected actions
+	ErrAssetNotFound      = 18 // requested file not found
 )
 
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
-	render.Status(r, code)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
+func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
+	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
+	render.Status(r, httpStatusCode)
+	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
 
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
+func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details string, errCode int) string {
 	uinfoStr := ""
 	if user, e := GetUserInfo(r); e == nil {
 		uinfoStr = user.Name + "/" + user.ID + " - "
@@ -40,5 +64,6 @@ func errDetailsMsg(r *http.Request, code int, err error, details string) string
 	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
 		remoteIP = remoteIP[:pos]
 	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
+		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -17,7 +17,7 @@ func TestSendErrorJSON(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
+			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456", 123)
 			return
 		}
 		w.WriteHeader(404)
@@ -33,7 +33,7 @@ func TestSendErrorJSON(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
+	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
 func TestErrorDetailsMsg(t *testing.T) {
@@ -41,8 +41,8 @@ func TestErrorDetailsMsg(t *testing.T) {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
 		require.Nil(t, err)
 		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
 	}
 	callerFn()
 }
@@ -53,8 +53,8 @@ func TestErrorDetailsMsgWithUser(t *testing.T) {
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
 		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"context"
 	"encoding/base64"
 	"io"
 	"net/http"
@@ -8,7 +9,6 @@ import (
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
@@ -22,6 +22,7 @@ type Image struct {
 	RemarkURL string
 	RoutePath string
 	Enabled   bool
+	Timeout   time.Duration
 }
 
 // Convert all img src links without https to proxied links
@@ -38,64 +39,75 @@ func (p Image) Convert(commentHTML string) string {
 	return p.replace(commentHTML, imgs)
 }
 
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
+// Handler returns http handler respond to proxied request
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
+
 	if !p.Enabled {
-		return router
+		http.Error(w, "proxy disabled", http.StatusNotImplemented)
+		return
 	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
-			return
-		}
 
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(func() error {
-			var e error
-			resp, e = client.Get(string(src))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
+		return
+	}
 
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
+	timeout := 60 * time.Second // default
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
 
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	client := http.Client{Timeout: 30 * time.Second}
+	var resp *http.Response
+	err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
+		var e error
+		req, e := http.NewRequest("GET", string(src), nil)
+		if e != nil {
+			return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
 		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
 	})
-	return router
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
+		return
+	}
+	defer func() {
+		if e := resp.Body.Close(); e != nil {
+			log.Printf("[WARN] can't close body, %s", e)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		w.WriteHeader(resp.StatusCode)
+		return
+	}
+
+	for k, v := range resp.Header {
+		if strings.EqualFold(k, "Content-Type") {
+			w.Header().Set(k, v[0])
+		}
+		if strings.EqualFold(k, "Content-Length") {
+			w.Header().Set(k, v[0])
+		}
+	}
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if _, e := io.Copy(w, resp.Body); e != nil {
+		log.Printf("[WARN] can't copy image stream, %s", e)
+	}
 }
 
 // extract gets all non-https images and return list of src

backend/app/rest/proxy/image_test.go

@@ -3,9 +3,13 @@ package proxy
 import (
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -45,9 +49,11 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{Enabled: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			res, err := img.extract(tt.inp)
+			assert.Nil(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
@@ -60,12 +66,11 @@ func TestPicture_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
@@ -87,6 +92,24 @@ func TestImage_Routes(t *testing.T) {
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
+func TestImage_RoutesTimedOut(t *testing.T) {
+	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	t.Log(string(b))
+	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
+}
+
 func TestPicture_Convert(t *testing.T) {
 	img := Image{Enabled: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
@@ -110,7 +133,13 @@ func imgHTTPServer(t *testing.T) *httptest.Server {
 			t.Log("http img request", r.URL)
 			w.Header().Add("Content-Length", "123")
 			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			assert.NoError(t, err)
+			return
+		}
+		if r.URL.Path == "/image/img-slow.png" {
+			time.Sleep(500 * time.Millisecond)
+			w.WriteHeader(500)
 			return
 		}
 		t.Log("http img request - not found", r.URL)

backend/app/store/comment.go

@@ -10,19 +10,21 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID        string          `json:"id" bson:"_id"`
-	ParentID  string          `json:"pid"`
-	Text      string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
-	Locator   Locator         `json:"locator"`
-	Score     int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
-	Timestamp time.Time       `json:"time" bson:"time"`
-	Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	Pin       bool            `json:"pin,omitempty" bson:"pin,omitempty"`
-	Deleted   bool            `json:"delete,omitempty" bson:"delete"`
-	PostTitle string          `json:"title,omitempty" bson:"title"`
+	ID          string          `json:"id" bson:"_id"`
+	ParentID    string          `json:"pid"`
+	Text        string          `json:"text"`
+	Orig        string          `json:"orig,omitempty"`
+	User        User            `json:"user"`
+	Locator     Locator         `json:"locator"`
+	Score       int             `json:"score"`
+	Votes       map[string]bool `json:"votes,omitempty"`
+	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
+	Controversy float64         `json:"controversy,omitempty"`
+	Timestamp   time.Time       `json:"time" bson:"time"`
+	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string          `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post

backend/app/store/engine/bolt_accessor.go

@@ -8,7 +8,7 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -74,6 +74,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
 	}
 	return &result, nil
 }
@@ -155,7 +156,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 
 		return bucket.ForEach(func(k, v []byte) error {
 			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
+			if e = json.Unmarshal(v, &comment); e != nil {
 				return errors.Wrap(e, "failed to unmarshal")
 			}
 			comments = append(comments, comment)
@@ -163,7 +164,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 		})
 	})
 
-	comments = sortComments(comments, sortFld)
+	comments = SortComments(comments, sortFld)
 	return comments, err
 }
 
@@ -195,7 +196,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
 			}
 
 			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
+			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
 				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
 				continue
 			}
@@ -335,11 +336,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.
 
 	// retrieve comments for refs
 	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
 		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
+		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
 			comments = append(comments, c)
 		}
 	}

backend/app/store/engine/bolt_accessor_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -16,8 +16,8 @@ import (
 var testDb = "test-remark.db"
 
 func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -37,8 +37,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
 }
 
 func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	comment := store.Comment{
 		ID:        "id-ro",
@@ -61,8 +61,8 @@ func TestBoltDB_CreateReadOnly(t *testing.T) {
 }
 
 func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -80,8 +80,9 @@ func TestBoltDB_Get(t *testing.T) {
 }
 
 func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
+
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
 	assert.Nil(t, err)
@@ -107,8 +108,8 @@ func TestBoltDB_Put(t *testing.T) {
 }
 
 func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.Last("radio-t", 0)
 	assert.Nil(t, err)
@@ -125,8 +126,8 @@ func TestBoltDB_Last(t *testing.T) {
 }
 
 func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -141,8 +142,8 @@ func TestBoltDB_Count(t *testing.T) {
 }
 
 func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	// add one more for https://radio-t.com/2
 	comment := store.Comment{
@@ -181,8 +182,8 @@ func TestBoltDB_List(t *testing.T) {
 }
 
 func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
 
@@ -224,8 +225,8 @@ func TestBoltDB_Info(t *testing.T) {
 }
 
 func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
 
 	res, err := b.User("radio-t", "user1", 5, 0)
 	assert.Nil(t, err)
@@ -253,7 +254,11 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 	os.Remove(testDb)
 	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
-	defer os.Remove(testDb)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}()
 
 	c := store.Comment{
 		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -303,8 +308,9 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 }
 
 func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+	var b, teardown = prep(t)
+	defer teardown()
+
 	count, err := b.UserCount("radio-t", "user1")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, count)
@@ -343,12 +349,12 @@ func TestBoltDB_New(t *testing.T) {
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	os.Remove(testDb)
 
 	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	assert.Nil(t, err)
-	b := boltStore
+	b = boltStore
 
 	comment := store.Comment{
 		ID:        "id-1",
@@ -370,5 +376,9 @@ func prep(t *testing.T) *BoltDB {
 	_, err = b.Create(comment)
 	assert.Nil(t, err)
 
-	return b
+	teardown = func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}
+	return b, teardown
 }

backend/app/store/engine/bolt_admin.go

@@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
 		}
 
 		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
+		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
 			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
+		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
 			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
 		}
 
 		// delete from "last" bucket
 		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
+		if err = lastBkt.Delete([]byte(commentID)); err != nil {
 			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
@@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
 			return nil
 		}
 
-		until, err := time.Parse(tsNano, string(val))
-		if err != nil {
+		until, e := time.Parse(tsNano, string(val))
+		if e != nil {
 			blocked = false
 			return nil
 		}
@@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(blocksBucketName))
 		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
+			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+			if errParse != nil {
+				return errors.Wrap(errParse, "can't parse block ts")
 			}
 			if time.Now().Before(ts) {
 				// get user name from comment user section
 				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
+				userComments, errUser := b.User(siteID, string(k), 1, 0)
+				if errUser == nil && len(userComments) > 0 {
 					userName = userComments[0].User.Name
 				}
 				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})

backend/app/store/engine/bolt_admin_test.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	"os"
 	"testing"
 	"time"
 
@@ -12,8 +11,9 @@ import (
 )
 
 func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -58,8 +58,9 @@ func TestBoltAdmin_Delete(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -78,8 +79,9 @@ func TestBoltAdmin_DeleteHard(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -102,8 +104,10 @@ func TestBoltAdmin_DeleteAll(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
+
 	err := b.DeleteUser("radio-t", "user1")
 	require.NoError(t, err)
 
@@ -130,8 +134,9 @@ func TestBoltAdmin_DeleteUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 
@@ -150,8 +155,10 @@ func TestBoltAdmin_BlockUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
+
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
 	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
@@ -160,8 +167,9 @@ func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
@@ -186,8 +194,9 @@ func TestBoltAdmin_BlockList(t *testing.T) {
 }
 
 func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
 
@@ -206,8 +215,9 @@ func TestBoltAdmin_ReadOnly(t *testing.T) {
 }
 
 func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
+
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
 

backend/app/store/engine/engine.go

@@ -62,8 +62,8 @@ const (
 	userLimit = 500
 )
 
-// sortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
+// SortComments is for engines can't sort data internally
+func SortComments(comments []store.Comment, sortFld string) []store.Comment {
 	sort.Slice(comments, func(i, j int) bool {
 		switch sortFld {
 		case "+time", "-time", "time", "+active", "-active", "active":
@@ -84,6 +84,18 @@ func sortComments(comments []store.Comment, sortFld string) []store.Comment {
 			}
 			return comments[i].Score < comments[j].Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortFld, "-") {
+				if comments[i].Controversy == comments[j].Controversy {
+					return comments[i].Timestamp.Before(comments[j].Timestamp)
+				}
+				return comments[i].Controversy > comments[j].Controversy
+			}
+			if comments[i].Controversy == comments[j].Controversy {
+				return comments[i].Timestamp.Before(comments[j].Timestamp)
+			}
+			return comments[i].Controversy < comments[j].Controversy
+
 		default:
 			return comments[i].Timestamp.Before(comments[j].Timestamp)
 		}

backend/app/store/engine/engine_test.go

@@ -11,33 +11,45 @@ import (
 
 func TestEngine_sortComments(t *testing.T) {
 	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
+		{ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
+		{ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
+		{ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
+		{ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
 	}
 
-	sortComments(cc, "+time")
+	SortComments(cc, "+time")
 	assert.Equal(t, "1", cc[0].ID)
 	assert.Equal(t, "2", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-time")
+	SortComments(cc, "-time")
 	assert.Equal(t, "4", cc[0].ID)
 	assert.Equal(t, "3", cc[1].ID)
 	assert.Equal(t, "2", cc[2].ID)
 	assert.Equal(t, "1", cc[3].ID)
 
-	sortComments(cc, "score")
+	SortComments(cc, "score")
 	assert.Equal(t, "2", cc[0].ID)
 	assert.Equal(t, "1", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-score")
+	SortComments(cc, "-score")
 	assert.Equal(t, "3", cc[0].ID)
 	assert.Equal(t, "4", cc[1].ID)
 	assert.Equal(t, "1", cc[2].ID)
 	assert.Equal(t, "2", cc[3].ID)
+
+	SortComments(cc, "controversy")
+	assert.Equal(t, "1", cc[0].ID)
+	assert.Equal(t, "4", cc[1].ID)
+	assert.Equal(t, "2", cc[2].ID)
+	assert.Equal(t, "3", cc[3].ID)
+
+	SortComments(cc, "-controversy")
+	assert.Equal(t, "3", cc[0].ID)
+	assert.Equal(t, "2", cc[1].ID)
+	assert.Equal(t, "1", cc[2].ID)
+	assert.Equal(t, "4", cc[3].ID)
 }

backend/app/store/engine/mongo.go

@@ -6,7 +6,7 @@ import (
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"github.com/go-pkgz/mongo"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -39,7 +39,7 @@ type metaUser struct {
 }
 
 // NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
+// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached.
 // important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
 func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
 	writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
@@ -230,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	for _, m := range metas {
-		ids = append(ids, m.ID)
+	for _, meta := range metas {
+		ids = append(ids, meta.ID)
 	}
 	return ids, nil
 }

backend/app/store/image/fs_store.go

@@ -0,0 +1,177 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"io/ioutil"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+	MaxHeight  int
+	MaxWidth   int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	data, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
+	}
+	if len(data) > f.MaxSize {
+		return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
+	}
+
+	// read header first, needs it to check if data is valid png/gif/jpeg
+	if !isValidImage(data[:512]) {
+		return "", errors.Errorf("file %s is not in allowed format", fileName)
+	}
+
+	data, resized := resize(data, f.MaxWidth, f.MaxHeight)
+
+	id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+	if resized { // resized also converted to png
+		id = strings.TrimSuffix(id, filepath.Ext(id)) + ".png"
+		dst = f.location(f.Staging, id)
+	}
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, fileName, len(data))
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", fpath, age)
+			rmErr := os.Remove(fpath)
+			_ = os.Remove(path.Dir(fpath)) // try to remove directory
+			return rmErr
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// SizeLimit returns max size of allowed image
+func (f *FileSystem) SizeLimit() int {
+	return f.MaxSize
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}

backend/app/store/image/fs_store_test.go

@@ -0,0 +1,312 @@
+package image
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveWithResize(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 32, 32
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1142, len(data))
+}
+
+func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 400, 300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 10786, len(data))
+}
+
+func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 1400, 1300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".jpg")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 23983, len(data))
+}
+
+func TestFsStore_WrongFormat(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
+	assert.EqualError(t, err, "file file1.png is not in allowed format")
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 2000
+	_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string) (path string) {
+		id, err := svc.Save(file, user, gopherPNG())
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		assert.Equal(t, 1462, len(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(path.Dir(img1))
+	assert.NotNil(t, err, "no dir %s on staging anymore", path.Dir(img1))
+
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = &FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    1500,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/image.go

@@ -0,0 +1,201 @@
+// Package image handles storing, resizing and retrieval of images
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
+package image
+
+//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
+
+import (
+	"bytes"
+	"context"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	"image/png"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/rs/xid"
+	"golang.org/x/image/draw"
+)
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+	SizeLimit() int                                                          // max image size
+}
+
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	idsFn func() (ids []string)
+	TS    time.Time
+}
+
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil || s == nil {
+		return
+	}
+
+	s.once.Do(func() {
+		log.Printf("[DEBUG] image submitter activated")
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
+				}
+			}
+			log.Printf("[INFO] image submitter terminated")
+		}()
+	})
+
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
+}
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
+		}
+	}
+}
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
+// biggest side (width or height) preserving aspect ratio.
+// Returns original data if resizing is not needed or failed.
+// If resized the result will be for png format and ok flag will be true.
+func resize(data []byte, limitW, limitH int) ([]byte, bool) {
+	if data == nil || limitW <= 0 || limitH <= 0 {
+		return data, false
+	}
+
+	src, _, err := image.Decode(bytes.NewBuffer(data))
+	if err != nil {
+		log.Printf("[WARN] can't decode image, %s", err)
+		return data, false
+	}
+
+	bounds := src.Bounds()
+	w, h := bounds.Dx(), bounds.Dy()
+	if w <= limitW && h <= limitH || w <= 0 || h <= 0 {
+		log.Printf("[DEBUG] resizing image is smaller that the limit or has 0 size")
+		return data, false
+	}
+
+	newW, newH := getProportionalSizes(w, h, limitW, limitH)
+	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
+	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+
+	var out bytes.Buffer
+	if err = png.Encode(&out, m); err != nil {
+		log.Printf("[WARN] can't encode resized image to png, %s", err)
+		return data, false
+	}
+	return out.Bytes(), true
+}
+
+// getProportionalSizes returns width and height resized by both dimensions proportionally
+func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
+
+	if srcW <= limitW && srcH <= limitH {
+		return srcW, srcH
+	}
+
+	ratioW := float64(srcW) / float64(limitW)
+	propH := float64(srcH) / ratioW
+
+	ratioH := float64(srcH) / float64(limitH)
+	propW := float64(srcW) / ratioH
+
+	if int(propH) > limitH {
+		return int(propW), limitH
+	}
+
+	return limitW, int(propH)
+}
+
+// check if file f is a valid image format, i.e. gif, png, jpeg or webp
+func isValidImage(b []byte) bool {
+	ct := http.DetectContentType(b)
+	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
+}
+
+// guid makes a globally unique id
+func guid() string {
+	return xid.New().String()
+}

backend/app/store/image/image_mock.go

@@ -0,0 +1,105 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package image
+
+import context "context"
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import time "time"
+
+// MockStore is an autogenerated mock type for the Store type
+type MockStore struct {
+	mock.Mock
+}
+
+// Cleanup provides a mock function with given fields: ctx, ttl
+func (_m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
+	ret := _m.Called(ctx, ttl)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, time.Duration) error); ok {
+		r0 = rf(ctx, ttl)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Commit provides a mock function with given fields: id
+func (_m *MockStore) Commit(id string) error {
+	ret := _m.Called(id)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(id)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Load provides a mock function with given fields: id
+func (_m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	ret := _m.Called(id)
+
+	var r0 io.ReadCloser
+	if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(io.ReadCloser)
+		}
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(string) int64); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string) error); ok {
+		r2 = rf(id)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: fileName, userID, r
+func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) {
+	ret := _m.Called(fileName, userID, r)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok {
+		r0 = rf(fileName, userID, r)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok {
+		r1 = rf(fileName, userID, r)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SizeLimit provides a mock function with given fields:
+func (_m *MockStore) SizeLimit() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}

backend/app/store/image/image_test.go

@@ -0,0 +1,146 @@
+package image
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"io/ioutil"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{ImageAPI: "/blah/"}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
+func TestService_Cleanup(t *testing.T) {
+	store := MockStore{}
+	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
+
+	svc := Service{Store: &store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+	store.AssertNumberOfCalls(t, "Cleanup", 10)
+}
+
+func TestService_Submit(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 0)
+	time.Sleep(time.Millisecond * 150)
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_Close(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 3)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_resize(t *testing.T) {
+
+	// Reader is nil.
+	resized, ok := resize(nil, 100, 100)
+	assert.Nil(t, resized)
+	assert.False(t, ok)
+
+	// Negative limit error.
+	resized, ok = resize([]byte("some picture bin data"), -1, -1)
+	require.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("some picture bin data"))
+	assert.False(t, ok)
+
+	// Decode error.
+	resized, ok = resize([]byte("invalid image content"), 100, 100)
+	assert.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("invalid image content"))
+	assert.False(t, ok)
+
+	cases := []struct {
+		file   string
+		wr, hr int
+	}{
+		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
+		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
+	}
+
+	for _, c := range cases {
+		img, err := ioutil.ReadFile(c.file)
+		require.Nil(t, err, "can't open test file %s", c.file)
+
+		// No need for resize, image dimensions are smaller than resize limit.
+		resized, ok = resize(img, 800, 800)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.Equal(t, resized, img)
+		assert.False(t, ok)
+
+		// Resizing to half of width. Check resized image format PNG.
+		resized, ok = resize(img, 400, 400)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.True(t, ok)
+
+		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
+		assert.Nil(t, err, "file %s", c.file)
+		assert.Equal(t, "png", format, "file %s", c.file)
+		bounds := imgRz.Bounds()
+		assert.Equal(t, c.wr, bounds.Dx(), "file %s", c.file)
+		assert.Equal(t, c.hr, bounds.Dy(), "file %s", c.file)
+	}
+
+}
+
+func TestGetProportionalSizes(t *testing.T) {
+	tbl := []struct {
+		inpW, inpH     int
+		limitW, limitH int
+		resW, resH     int
+	}{
+		{10, 20, 50, 25, 10, 20},
+		{400, 200, 50, 25, 50, 25},
+		{100, 100, 50, 25, 25, 25},
+		{100, 200, 50, 25, 12, 25},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			resW, resH := getProportionalSizes(tt.inpW, tt.inpH, tt.limitW, tt.limitH)
+			assert.Equal(t, tt.resW, resW, "width")
+			assert.Equal(t, tt.resH, resH, "height")
+		})
+	}
+}

backend/app/store/service/restricted_words_test.go

@@ -1,8 +1,9 @@
 package service
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMatcher_Tokenize(t *testing.T) {

backend/app/store/service/service.go

@@ -1,7 +1,9 @@
 package service
 
 import (
+	"math"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -14,6 +16,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // DataStore wraps store.Interface with additional methods
@@ -23,8 +26,10 @@ type DataStore struct {
 	AdminStore             admin.Store
 	MaxCommentSize         int
 	MaxVotes               int
+	PositiveScore          bool
 	TitleExtractor         *TitleExtractor
 	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -75,18 +80,44 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", ErrRestrictedWordsFound
 	}
 
-	// keep input title and set to extracted if missing
-	if s.TitleExtractor != nil && comment.PostTitle == "" {
-		if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
-			comment.PostTitle = title
-		} else {
-			log.Printf("[WARN] failed to set title, %v", err)
+	func() { // keep input title and set to extracted if missing
+		if s.TitleExtractor == nil || comment.PostTitle != "" {
+			return
 		}
-	}
+		title, e := s.TitleExtractor.Get(comment.Locator.URL)
+		if e != nil {
+			log.Printf("[WARN] failed to set title, %v", e)
+			return
+		}
+		comment.PostTitle = title
+	}()
 
+	s.submitImages(comment)
 	return s.Interface.Create(comment)
 }
 
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			return nil
+		}
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+}
+
 // prepareNewComment sets new comment fields, hashing and sanitizing data
 func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) {
 	// fill ID and time if empty
@@ -123,8 +154,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool)
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
 	comment, err = s.Get(locator, commentID)
@@ -154,6 +185,10 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		return comment, errors.Errorf("maximum number of votes exceeded for comment %s", commentID)
 	}
 
+	if s.PositiveScore && comment.Score <= 0 && !val {
+		return comment, errors.Errorf("minimal score reached for comment %s", commentID)
+	}
+
 	// reset vote if user changed to opposite
 	if voted && v != val {
 		delete(comment.Votes, userID)
@@ -171,9 +206,36 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		comment.Score--
 	}
 
+	comment.Vote = 0
+	if vv, ok := comment.Votes[userID]; ok {
+		if vv {
+			comment.Vote = 1
+		} else {
+			comment.Vote = -1
+		}
+	}
+
+	comment.Controversy = s.controversy(s.upsAndDowns(comment))
+
 	return comment, s.Put(locator, comment)
 }
 
+// controversy calculates controversial index of votes
+// source - https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L60
+func (s *DataStore) controversy(ups, downs int) float64 {
+
+	if downs <= 0 || ups <= 0 {
+		return 0
+	}
+
+	magnitude := ups + downs
+	balance := float64(downs) / float64(ups)
+	if ups <= downs {
+		balance = float64(ups) / float64(downs)
+	}
+	return math.Pow(float64(magnitude), balance)
+}
+
 // EditRequest contains fields needed for comment update
 type EditRequest struct {
 	Text    string
@@ -390,8 +452,45 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
 	return errs.ErrorOrNil()
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+// Find wraps engine's Find call and alter results if needed
+func (s *DataStore) Find(locator store.Locator, sort string) ([]store.Comment, error) {
+	comments, err := s.Interface.Find(locator, sort)
+	if err != nil {
+		return comments, err
+	}
+
+	changedSort := false
+	// set votes controversy for comments added prior to #274
+	for i, c := range comments {
+		if c.Controversy == 0 && len(c.Votes) > 0 {
+			comments[i].Controversy = s.controversy(s.upsAndDowns(c))
+			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+				changedSort = true
+			}
+		}
+	}
+
+	// resort commits if altered
+	if changedSort {
+		comments = engine.SortComments(comments, sort)
+	}
+
+	return comments, nil
+}
+
+func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
+	for _, v := range c.Votes {
+		if v {
+			ups++
+			continue
+		}
+		downs++
+	}
+	return ups, downs
+}
+
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()

backend/app/store/service/service_test.go

@@ -13,19 +13,22 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 var testDb = "/tmp/test-remark.db"
 
 func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -45,11 +48,11 @@ func TestService_CreateFromEmpty(t *testing.T) {
 	assert.Equal(t, "user", res.User.ID)
 	assert.Equal(t, "name", res.User.Name)
 	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, map[string]bool{}, res.Votes)
+	assert.Equal(t, map[string]bool(nil), res.Votes)
 }
 
 func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -76,7 +79,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
@@ -106,7 +109,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 }
 
 func TestService_SetTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -114,11 +117,13 @@ func TestService_SetTitle(t *testing.T) {
 			w.WriteHeader(404)
 		}
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -145,6 +150,8 @@ func TestService_SetTitle(t *testing.T) {
 	t.Logf("%+v", res)
 	assert.Equal(t, "", res.PostTitle)
 
+	b.TitleExtractor.cache.Purge()
+
 	atomic.StoreInt32(&titleEnable, 1)
 	c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
 	require.NoError(t, err)
@@ -156,7 +163,7 @@ func TestService_SetTitle(t *testing.T) {
 }
 
 func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -172,11 +179,13 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, c.Score)
+	assert.Equal(t, 1, c.Vote)
 	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
 
 	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
@@ -190,6 +199,8 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 1, res[0].Score)
+	assert.Equal(t, 1, res[0].Vote)
+	assert.Equal(t, 0.0, res[0].Controversy)
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
 	assert.Nil(t, err, "vote reset")
@@ -197,11 +208,12 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "vote reset ok")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "vote reset ok")
 }
 
 func TestService_VoteLimit(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -219,7 +231,7 @@ func TestService_VoteLimit(t *testing.T) {
 }
 
 func TestService_VotesDisabled(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -227,7 +239,7 @@ func TestService_VotesDisabled(t *testing.T) {
 }
 
 func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -243,7 +255,7 @@ func TestService_VoteAggressive(t *testing.T) {
 	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	// add a vote as user2
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
@@ -255,7 +267,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 		}()
 	}
 	wg.Wait()
@@ -274,7 +286,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
 		}()
 	}
 	wg.Wait()
@@ -287,7 +299,7 @@ func TestService_VoteAggressive(t *testing.T) {
 
 func TestService_VoteConcurrent(t *testing.T) {
 
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -304,21 +316,92 @@ func TestService_VoteConcurrent(t *testing.T) {
 	var wg sync.WaitGroup
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		i := i
+		ii := i
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
+				fmt.Sprintf("user1-%d", ii), true)
 		}()
 	}
 	wg.Wait()
 	res, err = b.Last("radio-t", 0)
 	require.NoError(t, err)
-	assert.Equal(t, 100, res[0].Score, "should have 1000 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 1000 votes")
+	assert.Equal(t, 100, res[0].Score, "should have 100 score")
+	assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
+	assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
+}
+
+func TestService_VotePositive(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: true}
+
+	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.EqualError(t, err, "minimal score reached for comment id-1")
+
+	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
+	assert.Nil(t, err, "minimal score doesn't affect positive vote")
+
+	b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: false}
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.Nil(t, err, "minimal score ignored")
+	assert.Equal(t, -1, c.Score)
+	assert.Equal(t, 0.0, c.Controversy)
+}
+
+func TestService_VoteControversy(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
+	assert.NoError(t, err)
+	assert.Equal(t, -1, c.Score, "should have -1 score")
+	assert.InDelta(t, 0.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user3", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c.Score, "should have 0 score")
+	assert.InDelta(t, 2.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c.Score, "should have 1 score")
+	assert.InDelta(t, 1.73, c.Controversy, 0.01)
+
+	// check if stored
+	res, err := b.Last("radio-t", 0)
+	require.NoError(t, err)
+	assert.Equal(t, 1, res[0].Score, "should have 1 score")
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+}
+
+func TestService_Controversy(t *testing.T) {
+	tbl := []struct {
+		ups, downs int
+		res        float64
+	}{
+		{0, 0, 0},
+		{10, 5, 3.87},
+		{20, 5, 2.24},
+		{20, 50, 5.47},
+		{20, 0, 0},
+		{1100, 500, 28.60},
+		{1100, 12100, 2.37},
+		{100, 100, 200},
+		{101, 101, 202},
+	}
+
+	b := DataStore{}
+	for i, tt := range tbl {
+		t.Run(fmt.Sprintf("check-%d-%d:%d", i, tt.ups, tt.downs), func(t *testing.T) {
+			assert.InDelta(t, tt.res, b.controversy(tt.ups, tt.downs), 0.01)
+		})
+	}
 }
 
 func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -342,7 +425,7 @@ func TestService_Pin(t *testing.T) {
 }
 
 func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -369,7 +452,7 @@ func TestService_EditComment(t *testing.T) {
 }
 
 func TestService_DeleteComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -388,7 +471,7 @@ func TestService_DeleteComment(t *testing.T) {
 }
 
 func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -405,7 +488,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
 }
 
 func TestService_EditCommentReplyFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -456,7 +539,7 @@ func TestService_ValidateComment(t *testing.T) {
 }
 
 func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := prepStoreEngine(t) // two comments for https://radio-t.com
 
 	// add one more for https://radio-t.com/2
@@ -485,7 +568,7 @@ func TestService_Counts(t *testing.T) {
 }
 
 func TestService_GetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -516,7 +599,7 @@ func TestService_GetMetas(t *testing.T) {
 }
 
 func TestService_SetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -540,7 +623,7 @@ func TestService_SetMetas(t *testing.T) {
 }
 
 func TestService_IsAdmin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
@@ -550,7 +633,7 @@ func TestService_IsAdmin(t *testing.T) {
 }
 
 func TestService_HasReplies(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
@@ -579,9 +662,69 @@ func TestService_HasReplies(t *testing.T) {
 	assert.True(t, b.HasReplies(comment))
 }
 
+func TestService_Find(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "123456",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+		Score:     1,
+		Votes:     map[string]bool{"id-1": true, "id-2": true, "123456": false},
+	}
+	_, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy
+	assert.Nil(t, err)
+
+	// make sure Controversy altered
+	res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy")
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, "123456", res[0].ID)
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+	assert.Equal(t, "id-1", res[1].ID)
+	assert.InDelta(t, 0, res[1].Controversy, 0.01)
+}
+
+func TestService_submitImages(t *testing.T) {
+	defer teardown(t)
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	mockStore := image.MockStore{}
+	mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil)
+	imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
+
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+
+	b.submitImages(c)
+	time.Sleep(250 * time.Millisecond)
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
+	_ = os.Remove(testDb)
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -609,3 +752,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 
 	return b
 }
+
+func teardown(_ *testing.T) {
+	_ = os.Remove(testDb)
+}

backend/app/store/service/title.go

@@ -3,19 +3,23 @@ package service
 import (
 	"io"
 	"net/http"
+	"time"
 
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 	"golang.org/x/net/html"
 )
 
-const teMaxCachedRecs = 1000
+const (
+	teCacheMaxRecs = 1000
+	teCacheTTL     = 15 * time.Minute
+)
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
 	client http.Client
-	cache  cache.LoadingCache
+	cache  lcw.LoadingCache
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
@@ -24,23 +28,27 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {
 		client: client,
 	}
 	var err error
-	res.cache, err = cache.NewMemoryCache(cache.MaxKeys(teMaxCachedRecs))
+	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
-		log.Printf("[WARN] failed to make cache, %v", err)
-		res.cache = &cache.Nop{}
+		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
+		res.cache = &lcw.Nop{}
 	}
 	return &res
 }
 
 // Get page for url and return title
 func (t *TitleExtractor) Get(url string) (string, error) {
-
-	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
-		resp, err := t.client.Get(url)
+	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
+	b, err := t.cache.Get(url, func() (lcw.Value, error) {
+		resp, err := client.Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
-		defer resp.Body.Close() //nolint
+		defer func() {
+			if err = resp.Body.Close(); err != nil {
+				log.Printf("[WARN] failed to close title extractor body, %v", err)
+			}
+		}()
 		if resp.StatusCode != 200 {
 			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
 		}
@@ -49,14 +57,16 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if !ok {
 			return nil, errors.Errorf("can't get title for %s", url)
 		}
-		return []byte(title), nil
+		return title, nil
 	})
 
+	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
+		_, _ = t.cache.Get(url, func() (lcw.Value, error) { return "", nil })
 		return "", err
 	}
 
-	return string(b), nil
+	return b.(string), nil
 }
 
 // get title from body reader, traverse recursively

backend/app/store/service/title_test.go

@@ -1,14 +1,17 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/syncs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +44,8 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -55,9 +59,59 @@ func TestTitle_Get(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/good")
-		require.Nil(t, err)
-		assert.Equal(t, "blah 123", title)
+		r, e := ex.Get(ts.URL + "/good")
+		require.Nil(t, e)
+		assert.Equal(t, "blah 123", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
+
+func TestTitle_GetConcurrent(t *testing.T) {
+	body := ""
+	for n := 0; n < 1000; n++ {
+		body += "something something blah blah\n"
+	}
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.String(), "/good") {
+			atomic.AddInt32(&hits, 1)
+			_, err := w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			assert.NoError(t, err)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	g := syncs.NewSizedGroup(10)
+
+	for i := 0; i < 100; i++ {
+		ii := i
+		g.Go(func(_ context.Context) {
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(ii))
+			require.Nil(t, err)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(ii), title)
+		})
+	}
+	g.Wait()
+	assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
+}
+
+func TestTitle_GetFailed(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(404)
+	}))
+
+	_, err := ex.Get(ts.URL + "/bad")
+	require.NotNil(t, err)
+
+	for i := 0; i < 100; i++ {
+		r, e := ex.Get(ts.URL + "/bad")
+		require.Nil(t, e)
+		assert.Equal(t, "", r)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
+}

backend/app/store/service/tree.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"sort"
@@ -151,6 +151,18 @@ func (t *Tree) sortNodes(sortType string) {
 			}
 			return t.Nodes[i].Comment.Score < t.Nodes[j].Comment.Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortType, "-") {
+				if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+					return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+				}
+				return t.Nodes[i].Comment.Controversy > t.Nodes[j].Comment.Controversy
+			}
+			if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+				return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+			}
+			return t.Nodes[i].Comment.Controversy < t.Nodes[j].Comment.Controversy
+
 		default:
 			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
 		}

backend/app/store/service/tree_test.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"encoding/json"
@@ -96,15 +96,15 @@ func TestTreeSortNodes(t *testing.T) {
 	comments := []store.Comment{
 		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
+		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2, Controversy: 10},
+		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3, Controversy: 5},
 		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
 		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
 		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
 		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
 		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
+		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2, Controversy: 7},
 		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
 		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
@@ -140,6 +140,19 @@ func TestTreeSortNodes(t *testing.T) {
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
+	res = MakeTree(comments, "+controversy", 0)
+	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
+
+	res = MakeTree(comments, "-controversy", 0)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
+
 	res = MakeTree(comments, "undefined", 0)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)

backend/app/store/user.go

@@ -2,7 +2,7 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -41,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id)
+	return hashWithFallback(sha1.New(), id) // nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -0,0 +1,49 @@
+module github.com/umputun/remark/backend
+
+go 1.12
+
+require (
+	cloud.google.com/go v0.38.0 // indirect
+	github.com/PuerkitoBio/goquery v1.4.0
+	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/coreos/bbolt v1.3.2
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/didip/tollbooth v4.0.0+incompatible
+	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
+	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
+	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/cors v1.0.0
+	github.com/go-chi/render v1.0.0
+	github.com/go-pkgz/auth v0.5.2
+	github.com/go-pkgz/lcw v0.2.0
+	github.com/go-pkgz/lgr v0.6.2
+	github.com/go-pkgz/mongo v1.1.2
+	github.com/go-pkgz/repeater v1.1.1
+	github.com/go-pkgz/rest v1.4.1
+	github.com/go-pkgz/syncs v1.1.0
+	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
+	github.com/gorilla/feeds v1.1.0
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
+	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.8.1
+	github.com/rakyll/statik v0.1.3
+	github.com/rs/xid v1.2.1
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.3.0
+	go.etcd.io/bbolt v1.3.2 // indirect
+	golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529
+	golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec
+	golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect
+	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
+	gopkg.in/russross/blackfriday.v2 v2.0.0
+)

backend/go.sum

@@ -0,0 +1,182 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
+github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
+github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
+github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
+github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.5.2 h1:Sdu2K6iZxMDt5nQSAomEbzJhmIoUqNwfFMiJwYV8Xn4=
+github.com/go-pkgz/auth v0.5.2/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
+github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
+github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
+github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
+github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
+github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
+github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
+github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
+github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
+github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec h1:arXJwtMuk5vqI1NHX0UTnNw977rYk5Sl4jQqHj+hun4=
+golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 h1:6M3SDHlHHDCx2PcQw3S4KsR170vGqDhJDOmpVd4Hjak=
+golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
+golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
+gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

backend/remark.rest

@@ -1,9 +1,13 @@
 
 ### find request with tree
-GET {{host}}/api/v1/find?site={{site}}&sort=-active&format=tree&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=tree&url={{url}}
 
 ### find request with plain
-GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=plain&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+
+### find request with plain. dev token for secret=secret, not admin
+GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### last 50 comments
 GET {{host}}/api/v1/last/50?site={{site}}
@@ -41,7 +45,7 @@ Content-Type: application/json
 }
 
 ### update comment
-PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site=remark&url=https://radio-t.com/blah1
+PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site={{site}}&url=https://radio-t.com/blah1
 Content-Type: application/json
 
 {
@@ -50,22 +54,23 @@ Content-Type: application/json
 }
 
 ### pin comment
-PUT {{host}}/api/v1/admin/pin/3665976683?site=remark&url={{url}}&pin=1
+PUT {{host}}/api/v1/admin/pin/3665976683?site={{site}}&url={{url}}&pin=1
 
 ### vote for comment
-PUT {{host}}/api/v1/vote/3665976683?site=remark&url={{url}}&vote=1
+PUT http://127.0.0.1:8080/api/v1/vote/8a8c0b80-0d0a-41c3-84ad-f4034704e827?site={{site}}&url={{url}}&vote=-1
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### get user info
 GET {{host}}/api/v1/user
 
 ### get comment by id
-GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
+GET {{host}}/api/v1/id/73e346f4-d57d-41a8-8803-6671aa187d8e?site={{site}}&url={{url}}
 
 ### get comment by id 2
-GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
+GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site={{site}}&url=https://radio-t.com/blah1
 
 ### get comment by user id
-GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
+GET {{host}}/api/v1/comments?site={{site}}&user={{user}}&limit=5
 
 ### get comment by user id2
 GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
@@ -84,7 +89,7 @@ Content-Type: application/json
 ]
 
 ### list commented posts
-GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
+GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=0
 
 ### block user
 PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1
@@ -107,7 +112,7 @@ GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
 ### site rss
 GET {{host}}/api/v1/rss/site?site={{site}}
 
-### replie rss
+### reply rss
 GET {{host}}/api/v1/rss/reply?site={{site}}&user={{user}}
 
 ### get default avatar
@@ -116,5 +121,14 @@ GET {{host}}/api/v1/avatar/blah
 ### get config
 GET  {{host}}/api/v1/config?site={{site}}
 
+### deleteme (use request). dev token for secret=secret, not admin
+POST {{host}}/api/v1/deleteme?site_id={{site}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
+
+### deletme, admin. admin token for secret=secret
+GET {{host}}/api/v1/admin/deleteme?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1NjU2NjI4MDYsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTU3NzEzOTQ2LCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiIiwiYXR0cnMiOnsiYWRtaW4iOmZhbHNlLCJibG9ja2VkIjpmYWxzZSwiZGVsZXRlX21lIjp0cnVlfX19.qmnsQt_jilHzoauA9D7t1m3w69qvAJsZkuKVPWhpdik
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE2NTc3MDQ5NzksImp0aSI6Ijk4ZDNhOGFkMGNjZWU5M2Q1MWYwYjJiOTY1ZjU2YmE2NmJkNmZiNzYiLCJpYXQiOjE1NTc3MDQ2NzksImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.YIt9Zq3n0O8PMkR78pGAqyHI0exCU0vLKjcHgcXfLFw
+
+
 ### ping
 GET {{host}}/ping

backend/vendor/cloud.google.com/go/AUTHORS

@@ -1,15 +0,0 @@
-# This is the official list of cloud authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as:
-# Name or Organization <email address>
-# The email address is not required for organizations.
-
-Filippo Valsorda <hi@filippo.io>
-Google Inc.
-Ingo Oeser <nightlyone@googlemail.com>
-Palm Stone Games, Inc.
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -1,39 +0,0 @@
-# People who have agreed to one of the CLAs and can contribute patches.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# https://developers.google.com/open-source/cla/individual
-# https://developers.google.com/open-source/cla/corporate
-#
-# Names should be added to this file as:
-#     Name <email address>
-
-# Keep the list alphabetically sorted.
-
-Alexis Hunt <lexer@google.com>
-Andreas Litt <andreas.litt@gmail.com>
-Andrew Gerrand <adg@golang.org>
-Brad Fitzpatrick <bradfitz@golang.org>
-Burcu Dogan <jbd@google.com>
-Dave Day <djd@golang.org>
-David Sansome <me@davidsansome.com>
-David Symonds <dsymonds@golang.org>
-Filippo Valsorda <hi@filippo.io>
-Glenn Lewis <gmlewis@google.com>
-Ingo Oeser <nightlyone@googlemail.com>
-Johan Euphrosine <proppy@google.com>
-Jonathan Amsterdam <jba@google.com>
-Kunpei Sakai <namusyaka@gmail.com>
-Luna Duclos <luna.duclos@palmstonegames.com>
-Magnus Hiie <magnus.hiie@gmail.com>
-Mario Castro <mariocaster@gmail.com>
-Michael McGreevy <mcgreevy@golang.org>
-Omar Jarjur <ojarjur@google.com>
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Sarah Adams <shadams@google.com>
-Thanatat Tamtan <acoshift@gmail.com>
-Toby Burress <kurin@google.com>
-Tuo Shan <shantuo@google.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -1,4 +1,4 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
+// Copyright 2014 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
 package metadata // import "cloud.google.com/go/compute/metadata"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -31,9 +32,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"golang.org/x/net/context"
-	"golang.org/x/net/context/ctxhttp"
 )
 
 const (
@@ -64,7 +62,7 @@ var (
 )
 
 var (
-	metaClient = &http.Client{
+	defaultClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
@@ -72,15 +70,15 @@ var (
 			}).Dial,
 			ResponseHeaderTimeout: 2 * time.Second,
 		},
-	}
-	subscribeClient = &http.Client{
+	}}
+	subscribeClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).Dial,
 		},
-	}
+	}}
 )
 
 // NotDefinedError is returned when requested metadata is not defined.
@@ -95,74 +93,16 @@ func (suffix NotDefinedError) Error() string {
 	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
 }
 
-// Get returns a value from the metadata service.
-// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
-//
-// If the GCE_METADATA_HOST environment variable is not defined, a default of
-// 169.254.169.254 will be used instead.
-//
-// If the requested metadata is not defined, the returned error will
-// be of type NotDefinedError.
-func Get(suffix string) (string, error) {
-	val, _, err := getETag(metaClient, suffix)
-	return val, err
-}
-
-// getETag returns a value from the metadata service as well as the associated
-// ETag using the provided client. This func is otherwise equivalent to Get.
-func getETag(client *http.Client, suffix string) (value, etag string, err error) {
-	// Using a fixed IP makes it very difficult to spoof the metadata service in
-	// a container, which is an important use-case for local testing of cloud
-	// deployments. To enable spoofing of the metadata service, the environment
-	// variable GCE_METADATA_HOST is first inspected to decide where metadata
-	// requests shall go.
-	host := os.Getenv(metadataHostEnv)
-	if host == "" {
-		// Using 169.254.169.254 instead of "metadata" here because Go
-		// binaries built with the "netgo" tag and without cgo won't
-		// know the search suffix for "metadata" is
-		// ".google.internal", and this IP address is documented as
-		// being stable anyway.
-		host = metadataIP
-	}
-	url := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("Metadata-Flavor", "Google")
-	req.Header.Set("User-Agent", userAgent)
-	res, err := client.Do(req)
-	if err != nil {
-		return "", "", err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == http.StatusNotFound {
-		return "", "", NotDefinedError(suffix)
-	}
-	if res.StatusCode != 200 {
-		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
-	all, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", "", err
-	}
-	return string(all), res.Header.Get("Etag"), nil
-}
-
-func getTrimmed(suffix string) (s string, err error) {
-	s, err = Get(suffix)
-	s = strings.TrimSpace(s)
-	return
-}
-
-func (c *cachedValue) get() (v string, err error) {
+func (c *cachedValue) get(cl *Client) (v string, err error) {
 	defer c.mu.Unlock()
 	c.mu.Lock()
 	if c.v != "" {
 		return c.v, nil
 	}
 	if c.trim {
-		v, err = getTrimmed(c.k)
+		v, err = cl.getTrimmed(c.k)
 	} else {
-		v, err = Get(c.k)
+		v, err = cl.Get(c.k)
 	}
 	if err == nil {
 		c.v = v
@@ -197,11 +137,11 @@ func testOnGCE() bool {
 	resc := make(chan bool, 2)
 
 	// Try two strategies in parallel.
-	// See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194
+	// See https://github.com/googleapis/google-cloud-go/issues/194
 	go func() {
 		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
 		req.Header.Set("User-Agent", userAgent)
-		res, err := ctxhttp.Do(ctx, metaClient, req)
+		res, err := defaultClient.hc.Do(req.WithContext(ctx))
 		if err != nil {
 			resc <- false
 			return
@@ -266,6 +206,255 @@ func systemInfoSuggestsGCE() bool {
 	return name == "Google" || name == "Google Compute Engine"
 }
 
+// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no
+// ResponseHeaderTimeout).
+func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	return subscribeClient.Subscribe(suffix, fn)
+}
+
+// Get calls Client.Get on the default client.
+func Get(suffix string) (string, error) { return defaultClient.Get(suffix) }
+
+// ProjectID returns the current instance's project ID string.
+func ProjectID() (string, error) { return defaultClient.ProjectID() }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() }
+
+// InternalIP returns the instance's primary internal IP address.
+func InternalIP() (string, error) { return defaultClient.InternalIP() }
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func Hostname() (string, error) { return defaultClient.Hostname() }
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() }
+
+// InstanceID returns the current VM's numeric instance ID.
+func InstanceID() (string, error) { return defaultClient.InstanceID() }
+
+// InstanceName returns the current VM's instance ID string.
+func InstanceName() (string, error) { return defaultClient.InstanceName() }
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func Zone() (string, error) { return defaultClient.Zone() }
+
+// InstanceAttributes calls Client.InstanceAttributes on the default client.
+func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() }
+
+// ProjectAttributes calls Client.ProjectAttributes on the default client.
+func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() }
+
+// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client.
+func InstanceAttributeValue(attr string) (string, error) {
+	return defaultClient.InstanceAttributeValue(attr)
+}
+
+// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client.
+func ProjectAttributeValue(attr string) (string, error) {
+	return defaultClient.ProjectAttributeValue(attr)
+}
+
+// Scopes calls Client.Scopes on the default client.
+func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) }
+
+func strsContains(ss []string, s string) bool {
+	for _, v := range ss {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// A Client provides metadata.
+type Client struct {
+	hc *http.Client
+}
+
+// NewClient returns a Client that can be used to fetch metadata. All HTTP requests
+// will use the given http.Client instead of the default client.
+func NewClient(c *http.Client) *Client {
+	return &Client{hc: c}
+}
+
+// getETag returns a value from the metadata service as well as the associated ETag.
+// This func is otherwise equivalent to Get.
+func (c *Client) getETag(suffix string) (value, etag string, err error) {
+	// Using a fixed IP makes it very difficult to spoof the metadata service in
+	// a container, which is an important use-case for local testing of cloud
+	// deployments. To enable spoofing of the metadata service, the environment
+	// variable GCE_METADATA_HOST is first inspected to decide where metadata
+	// requests shall go.
+	host := os.Getenv(metadataHostEnv)
+	if host == "" {
+		// Using 169.254.169.254 instead of "metadata" here because Go
+		// binaries built with the "netgo" tag and without cgo won't
+		// know the search suffix for "metadata" is
+		// ".google.internal", and this IP address is documented as
+		// being stable anyway.
+		host = metadataIP
+	}
+	u := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, _ := http.NewRequest("GET", u, nil)
+	req.Header.Set("Metadata-Flavor", "Google")
+	req.Header.Set("User-Agent", userAgent)
+	res, err := c.hc.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return "", "", NotDefinedError(suffix)
+	}
+	all, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", err
+	}
+	if res.StatusCode != 200 {
+		return "", "", &Error{Code: res.StatusCode, Message: string(all)}
+	}
+	return string(all), res.Header.Get("Etag"), nil
+}
+
+// Get returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+func (c *Client) Get(suffix string) (string, error) {
+	val, _, err := c.getETag(suffix)
+	return val, err
+}
+
+func (c *Client) getTrimmed(suffix string) (s string, err error) {
+	s, err = c.Get(suffix)
+	s = strings.TrimSpace(s)
+	return
+}
+
+func (c *Client) lines(suffix string) ([]string, error) {
+	j, err := c.Get(suffix)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.Split(strings.TrimSpace(j), "\n")
+	for i := range s {
+		s[i] = strings.TrimSpace(s[i])
+	}
+	return s, nil
+}
+
+// ProjectID returns the current instance's project ID string.
+func (c *Client) ProjectID() (string, error) { return projID.get(c) }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) }
+
+// InstanceID returns the current VM's numeric instance ID.
+func (c *Client) InstanceID() (string, error) { return instID.get(c) }
+
+// InternalIP returns the instance's primary internal IP address.
+func (c *Client) InternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/ip")
+}
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func (c *Client) ExternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+}
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func (c *Client) Hostname() (string, error) {
+	return c.getTrimmed("instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func (c *Client) InstanceTags() ([]string, error) {
+	var s []string
+	j, err := c.Get("instance/tags")
+	if err != nil {
+		return nil, err
+	}
+	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// InstanceName returns the current VM's instance ID string.
+func (c *Client) InstanceName() (string, error) {
+	host, err := c.Hostname()
+	if err != nil {
+		return "", err
+	}
+	return strings.Split(host, ".")[0], nil
+}
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func (c *Client) Zone() (string, error) {
+	zone, err := c.getTrimmed("instance/zone")
+	// zone is of the form "projects/<projNum>/zones/<zoneName>".
+	if err != nil {
+		return "", err
+	}
+	return zone[strings.LastIndex(zone, "/")+1:], nil
+}
+
+// InstanceAttributes returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") }
+
+// ProjectAttributes returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") }
+
+// InstanceAttributeValue returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) InstanceAttributeValue(attr string) (string, error) {
+	return c.Get("instance/attributes/" + attr)
+}
+
+// ProjectAttributeValue returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) ProjectAttributeValue(attr string) (string, error) {
+	return c.Get("project/attributes/" + attr)
+}
+
+// Scopes returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) Scopes(serviceAccount string) ([]string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return c.lines("instance/service-accounts/" + serviceAccount + "/scopes")
+}
+
 // Subscribe subscribes to a value from the metadata service.
 // The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
 // The suffix may contain query parameters.
@@ -275,11 +464,11 @@ func systemInfoSuggestsGCE() bool {
 // and ok false. Subscribe blocks until fn returns a non-nil error or the value
 // is deleted. Subscribe returns the error value returned from the last call to
 // fn, which may be nil when ok == false.
-func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error {
 	const failedSubscribeSleep = time.Second * 5
 
 	// First check to see if the metadata value exists at all.
-	val, lastETag, err := getETag(subscribeClient, suffix)
+	val, lastETag, err := c.getETag(suffix)
 	if err != nil {
 		return err
 	}
@@ -295,7 +484,7 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 		suffix += "?wait_for_change=true&last_etag="
 	}
 	for {
-		val, etag, err := getETag(subscribeClient, suffix+url.QueryEscape(lastETag))
+		val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag))
 		if err != nil {
 			if _, deleted := err.(NotDefinedError); !deleted {
 				time.Sleep(failedSubscribeSleep)
@@ -311,127 +500,14 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 	}
 }
 
-// ProjectID returns the current instance's project ID string.
-func ProjectID() (string, error) { return projID.get() }
-
-// NumericProjectID returns the current instance's numeric project ID.
-func NumericProjectID() (string, error) { return projNum.get() }
-
-// InternalIP returns the instance's primary internal IP address.
-func InternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/ip")
+// Error contains an error response from the server.
+type Error struct {
+	// Code is the HTTP response status code.
+	Code int
+	// Message is the server response message.
+	Message string
 }
 
-// ExternalIP returns the instance's primary external (public) IP address.
-func ExternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
-}
-
-// Hostname returns the instance's hostname. This will be of the form
-// "<instanceID>.c.<projID>.internal".
-func Hostname() (string, error) {
-	return getTrimmed("instance/hostname")
-}
-
-// InstanceTags returns the list of user-defined instance tags,
-// assigned when initially creating a GCE instance.
-func InstanceTags() ([]string, error) {
-	var s []string
-	j, err := Get("instance/tags")
-	if err != nil {
-		return nil, err
-	}
-	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
-		return nil, err
-	}
-	return s, nil
-}
-
-// InstanceID returns the current VM's numeric instance ID.
-func InstanceID() (string, error) {
-	return instID.get()
-}
-
-// InstanceName returns the current VM's instance ID string.
-func InstanceName() (string, error) {
-	host, err := Hostname()
-	if err != nil {
-		return "", err
-	}
-	return strings.Split(host, ".")[0], nil
-}
-
-// Zone returns the current VM's zone, such as "us-central1-b".
-func Zone() (string, error) {
-	zone, err := getTrimmed("instance/zone")
-	// zone is of the form "projects/<projNum>/zones/<zoneName>".
-	if err != nil {
-		return "", err
-	}
-	return zone[strings.LastIndex(zone, "/")+1:], nil
-}
-
-// InstanceAttributes returns the list of user-defined attributes,
-// assigned when initially creating a GCE VM instance. The value of an
-// attribute can be obtained with InstanceAttributeValue.
-func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }
-
-// ProjectAttributes returns the list of user-defined attributes
-// applying to the project as a whole, not just this VM.  The value of
-// an attribute can be obtained with ProjectAttributeValue.
-func ProjectAttributes() ([]string, error) { return lines("project/attributes/") }
-
-func lines(suffix string) ([]string, error) {
-	j, err := Get(suffix)
-	if err != nil {
-		return nil, err
-	}
-	s := strings.Split(strings.TrimSpace(j), "\n")
-	for i := range s {
-		s[i] = strings.TrimSpace(s[i])
-	}
-	return s, nil
-}
-
-// InstanceAttributeValue returns the value of the provided VM
-// instance attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// InstanceAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func InstanceAttributeValue(attr string) (string, error) {
-	return Get("instance/attributes/" + attr)
-}
-
-// ProjectAttributeValue returns the value of the provided
-// project attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// ProjectAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func ProjectAttributeValue(attr string) (string, error) {
-	return Get("project/attributes/" + attr)
-}
-
-// Scopes returns the service account scopes for the given account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
-func Scopes(serviceAccount string) ([]string, error) {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return lines("instance/service-accounts/" + serviceAccount + "/scopes")
-}
-
-func strsContains(ss []string, s string) bool {
-	for _, v := range ss {
-		if v == s {
-			return true
-		}
-	}
-	return false
+func (e *Error) Error() string {
+	return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message)
 }

backend/vendor/github.com/coreos/bbolt/.gitignore

@@ -2,3 +2,4 @@
 *.test
 *.swp
 /bin/
+cover.out

backend/vendor/github.com/coreos/bbolt/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+go_import_path: go.etcd.io/bbolt
+
+sudo: false
+
+go:
+- 1.11
+
+before_install:
+- go get -v honnef.co/go/tools/...
+- go get -v github.com/kisielk/errcheck
+
+script:
+- make fmt
+- make test
+- make race
+# - make errcheck

backend/vendor/github.com/coreos/bbolt/Makefile

@@ -5,14 +5,34 @@ GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 default: build
 
 race:
-	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@TEST_FREELIST_TYPE=hashmap go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@echo "array freelist test"
+	@TEST_FREELIST_TYPE=array go test -v -race -test.run="TestSimulate_(100op|1000op)"
+
+fmt:
+	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
+
+# go get honnef.co/go/tools/simple
+gosimple:
+	gosimple ./...
+
+# go get honnef.co/go/tools/unused
+unused:
+	unused ./...
 
 # go get github.com/kisielk/errcheck
 errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/boltdb/bolt
+	@errcheck -ignorepkg=bytes -ignore=os:Remove go.etcd.io/bbolt
 
-test: 
-	@go test -v -cover .
-	@go test -v ./cmd/bolt
+test:
+	TEST_FREELIST_TYPE=hashmap go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	TEST_FREELIST_TYPE=hashmap go test -v ./cmd/bbolt
 
-.PHONY: fmt test
+	@echo "array freelist test"
+
+	@TEST_FREELIST_TYPE=array go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	@TEST_FREELIST_TYPE=array go test -v ./cmd/bbolt
+
+.PHONY: race fmt errcheck test gosimple unused

backend/vendor/github.com/coreos/bbolt/README.md

@@ -1,5 +1,18 @@
-Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg)
-====
+bbolt
+=====
+
+[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt)
+[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt)
+[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt)
+[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt)
+[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases)
+[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE)
+
+bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value
+store. The purpose of this fork is to provide the Go community with an active
+maintenance and development target for Bolt; the goal is improved reliability
+and stability. bbolt includes bug fixes, performance enhancements, and features
+not found in Bolt while preserving backwards compatibility with the Bolt API.
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,
@@ -10,47 +23,55 @@ Since Bolt is meant to be used as such a low-level piece of functionality,
 simplicity is key. The API will be small and only focus on getting values
 and setting values. That's it.
 
+[gh_ben]: https://github.com/benbjohnson
+[bolt]: https://github.com/boltdb/bolt
 [hyc_symas]: https://twitter.com/hyc_symas
 [lmdb]: http://symas.com/mdb/
 
 ## Project Status
 
-Bolt is stable and the API is fixed. Full unit test coverage and randomized
-black box testing are used to ensure database consistency and thread safety.
-Bolt is currently in high-load production environments serving databases as
-large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed
-services every day.
+Bolt is stable, the API is fixed, and the file format is fixed. Full unit
+test coverage and randomized black box testing are used to ensure database
+consistency and thread safety. Bolt is currently used in high-load production
+environments serving databases as large as 1TB. Many companies such as
+Shopify and Heroku use Bolt-backed services every day.
+
+## Project versioning
+
+bbolt uses [semantic versioning](http://semver.org).
+API should not change between patch and minor releases.
+New minor versions may add additional features to the API.
 
 ## Table of Contents
 
-- [Getting Started](#getting-started)
-  - [Installing](#installing)
-  - [Opening a database](#opening-a-database)
-  - [Transactions](#transactions)
-    - [Read-write transactions](#read-write-transactions)
-    - [Read-only transactions](#read-only-transactions)
-    - [Batch read-write transactions](#batch-read-write-transactions)
-    - [Managing transactions manually](#managing-transactions-manually)
-  - [Using buckets](#using-buckets)
-  - [Using key/value pairs](#using-keyvalue-pairs)
-  - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
-  - [Iterating over keys](#iterating-over-keys)
-    - [Prefix scans](#prefix-scans)
-    - [Range scans](#range-scans)
-    - [ForEach()](#foreach)
-  - [Nested buckets](#nested-buckets)
-  - [Database backups](#database-backups)
-  - [Statistics](#statistics)
-  - [Read-Only Mode](#read-only-mode)
-  - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
-- [Resources](#resources)
-- [Comparison with other databases](#comparison-with-other-databases)
-  - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
-  - [LevelDB, RocksDB](#leveldb-rocksdb)
-  - [LMDB](#lmdb)
-- [Caveats & Limitations](#caveats--limitations)
-- [Reading the Source](#reading-the-source)
-- [Other Projects Using Bolt](#other-projects-using-bolt)
+  - [Getting Started](#getting-started)
+    - [Installing](#installing)
+    - [Opening a database](#opening-a-database)
+    - [Transactions](#transactions)
+      - [Read-write transactions](#read-write-transactions)
+      - [Read-only transactions](#read-only-transactions)
+      - [Batch read-write transactions](#batch-read-write-transactions)
+      - [Managing transactions manually](#managing-transactions-manually)
+    - [Using buckets](#using-buckets)
+    - [Using key/value pairs](#using-keyvalue-pairs)
+    - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
+    - [Iterating over keys](#iterating-over-keys)
+      - [Prefix scans](#prefix-scans)
+      - [Range scans](#range-scans)
+      - [ForEach()](#foreach)
+    - [Nested buckets](#nested-buckets)
+    - [Database backups](#database-backups)
+    - [Statistics](#statistics)
+    - [Read-Only Mode](#read-only-mode)
+    - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
+  - [Resources](#resources)
+  - [Comparison with other databases](#comparison-with-other-databases)
+    - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
+    - [LevelDB, RocksDB](#leveldb-rocksdb)
+    - [LMDB](#lmdb)
+  - [Caveats & Limitations](#caveats--limitations)
+  - [Reading the Source](#reading-the-source)
+  - [Other Projects Using Bolt](#other-projects-using-bolt)
 
 ## Getting Started
 
@@ -59,13 +80,28 @@ services every day.
 To start using Bolt, install Go and run `go get`:
 
 ```sh
-$ go get github.com/boltdb/bolt/...
+$ go get go.etcd.io/bbolt/...
 ```
 
 This will retrieve the library and install the `bolt` command line utility into
 your `$GOBIN` path.
 
 
+### Importing bbolt
+
+To use bbolt as an embedded key-value store, import as:
+
+```go
+import bolt "go.etcd.io/bbolt"
+
+db, err := bolt.Open(path, 0666, nil)
+if err != nil {
+  return err
+}
+defer db.Close()
+```
+
+
 ### Opening a database
 
 The top-level object in Bolt is a `DB`. It is represented as a single file on
@@ -79,7 +115,7 @@ package main
 import (
 	"log"
 
-	"github.com/boltdb/bolt"
+	bolt "go.etcd.io/bbolt"
 )
 
 func main() {
@@ -209,7 +245,7 @@ and then safely close your transaction if an error is returned. This is the
 recommended way to use Bolt transactions.
 
 However, sometimes you may want to manually start and end your transactions.
-You can use the `Tx.Begin()` function directly but **please** be sure to close
+You can use the `DB.Begin()` function directly but **please** be sure to close
 the transaction.
 
 ```go
@@ -395,7 +431,7 @@ db.View(func(tx *bolt.Tx) error {
 	c := tx.Bucket([]byte("MyBucket")).Cursor()
 
 	prefix := []byte("1234")
-	for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
 		fmt.Printf("key=%s, value=%s\n", k, v)
 	}
 
@@ -448,6 +484,10 @@ db.View(func(tx *bolt.Tx) error {
 })
 ```
 
+Please note that keys and values in `ForEach()` are only valid while
+the transaction is open. If you need to use a key or value outside of
+the transaction, you must use `copy()` to copy it to another byte
+slice.
 
 ### Nested buckets
 
@@ -460,6 +500,55 @@ func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error)
 func (*Bucket) DeleteBucket(key []byte) error
 ```
 
+Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings.
+
+```go
+
+// createUser creates a new user in the given account.
+func createUser(accountID int, u *User) error {
+    // Start the transaction.
+    tx, err := db.Begin(true)
+    if err != nil {
+        return err
+    }
+    defer tx.Rollback()
+
+    // Retrieve the root bucket for the account.
+    // Assume this has already been created when the account was set up.
+    root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10)))
+
+    // Setup the users bucket.
+    bkt, err := root.CreateBucketIfNotExists([]byte("USERS"))
+    if err != nil {
+        return err
+    }
+
+    // Generate an ID for the new user.
+    userID, err := bkt.NextSequence()
+    if err != nil {
+        return err
+    }
+    u.ID = userID
+
+    // Marshal and save the encoded user.
+    if buf, err := json.Marshal(u); err != nil {
+        return err
+    } else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil {
+        return err
+    }
+
+    // Commit the transaction.
+    if err := tx.Commit(); err != nil {
+        return err
+    }
+
+    return nil
+}
+
+```
+
+
+
 
 ### Database backups
 
@@ -469,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block
 your other database reads and writes.
 
 By default, it will use a regular file handle which will utilize the operating
-system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx)
+system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx)
 documentation for information about optimizing for larger-than-RAM datasets.
 
 One common use case is to backup over HTTP so you can use tools like `cURL` to
@@ -715,6 +804,9 @@ Here are a few things to note when evaluating and using Bolt:
   can be reused by a new page or can be unmapped from virtual memory and you'll
   see an `unexpected fault address` panic when accessing it.
 
+* Bolt uses an exclusive write lock on the database file so it cannot be
+  shared by multiple processes.
+
 * Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
   buckets that have random inserts will cause your database to have very poor
   page utilization.
@@ -755,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt:
 
 ## Reading the Source
 
-Bolt is a relatively small code base (<3KLOC) for an embedded, serializable,
+Bolt is a relatively small code base (<5KLOC) for an embedded, serializable,
 transactional key/value database so it can be a good starting point for people
 interested in how databases work.
 
@@ -807,46 +899,56 @@ them via pull request.
 
 Below is a list of public, open source projects that use Bolt:
 
-* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
-* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
+* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
 * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
-* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
-* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
-* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
-* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
-* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
-* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
-* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
-* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
-* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
+* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal.
+* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support.
+* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB
 * [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
-* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
-* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
-* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
-* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
+* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
+* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
 * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
-* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
-* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
-* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
-* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
-* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
-* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
-* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
-* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
-* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
-  backed by boltdb.
+* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet.
 * [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining
   simple tx and key scans.
-* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
-* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
+* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
+* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
+* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
+* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency.
+* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
+* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
+* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
 * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
-* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
-* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
-* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
 * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
-* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
-* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
-* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
 * [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter.
+* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains
+* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
+* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
+* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
+* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
+* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies
+* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
+* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding.
+* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
+* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
+* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
+* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
+* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
+* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
+* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
+* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
+* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi.
+* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
+* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
+  backed by boltdb.
+* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
+* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
+* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
+* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
+* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
+* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development.
+* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
 
 If you are using Bolt in a project please send a pull request to add it to the list.

backend/vendor/github.com/coreos/bbolt/appveyor.yml

@@ -1,18 +0,0 @@
-version: "{build}"
-
-os: Windows Server 2012 R2
-
-clone_folder: c:\gopath\src\github.com\boltdb\bolt
-
-environment:
-  GOPATH: c:\gopath
-
-install:
-  - echo %PATH%
-  - echo %GOPATH%
-  - go version
-  - go env
-  - go get -v -t ./...
-
-build_script:
-  - go test -v ./...

backend/vendor/github.com/coreos/bbolt/bolt_386.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_amd64.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_arm.go

@@ -1,7 +1,28 @@
-package bolt
+package bbolt
+
+import "unsafe"
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned bool
+
+func init() {
+	// Simple check to see whether this arch handles unaligned load/stores
+	// correctly.
+
+	// ARM9 and older devices require load/stores to be from/to aligned
+	// addresses. If not, the lower 2 bits are cleared and that address is
+	// read in a jumbled up order.
+
+	// See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+
+	raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11}
+	val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2))
+
+	brokenUnaligned = val != 0x11222211
+}

backend/vendor/github.com/coreos/bbolt/bolt_arm64.go

@@ -1,9 +1,12 @@
 // +build arm64
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_linux.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"syscall"

backend/vendor/github.com/coreos/bbolt/bolt_mips64x.go

@@ -0,0 +1,12 @@
+// +build mips64 mips64le
+
+package bbolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x8000000000 // 512GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_mipsx.go

@@ -0,0 +1,12 @@
+// +build mips mipsle
+
+package bbolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x40000000 // 1GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_openbsd.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"syscall"

backend/vendor/github.com/coreos/bbolt/bolt_ppc.go

@@ -1,9 +1,12 @@
 // +build ppc
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_ppc64.go

@@ -1,9 +1,12 @@
 // +build ppc64
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_ppc64le.go

@@ -1,9 +1,12 @@
 // +build ppc64le
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_s390x.go

@@ -1,9 +1,12 @@
 // +build s390x
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_unix.go

@@ -1,41 +1,43 @@
 // +build !windows,!plan9,!solaris
 
-package bolt
+package bbolt
 
 import (
 	"fmt"
-	"os"
 	"syscall"
 	"time"
 	"unsafe"
 )
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	fd := db.file.Fd()
+	flag := syscall.LOCK_NB
+	if exclusive {
+		flag |= syscall.LOCK_EX
+	} else {
+		flag |= syscall.LOCK_SH
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		flag := syscall.LOCK_SH
-		if exclusive {
-			flag = syscall.LOCK_EX
-		}
-
-		// Otherwise attempt to obtain an exclusive lock.
-		err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB)
+		// Attempt to obtain an exclusive lock.
+		err := syscall.Flock(int(fd), flag)
 		if err == nil {
 			return nil
 		} else if err != syscall.EWOULDBLOCK {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
@@ -53,7 +55,9 @@ func mmap(db *DB, sz int) error {
 	}
 
 	// Advise the kernel that the mmap is accessed randomly.
-	if err := madvise(b, syscall.MADV_RANDOM); err != nil {
+	err = madvise(b, syscall.MADV_RANDOM)
+	if err != nil && err != syscall.ENOSYS {
+		// Ignore not implemented error in kernel because it still works.
 		return fmt.Errorf("madvise: %s", err)
 	}
 

backend/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go

@@ -1,8 +1,7 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
-	"os"
 	"syscall"
 	"time"
 	"unsafe"
@@ -11,36 +10,35 @@ import (
 )
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	fd := db.file.Fd()
+	var lockType int16
+	if exclusive {
+		lockType = syscall.F_WRLCK
+	} else {
+		lockType = syscall.F_RDLCK
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		var lock syscall.Flock_t
-		lock.Start = 0
-		lock.Len = 0
-		lock.Pid = 0
-		lock.Whence = 0
-		lock.Pid = 0
-		if exclusive {
-			lock.Type = syscall.F_WRLCK
-		} else {
-			lock.Type = syscall.F_RDLCK
-		}
-		err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock)
+		// Attempt to obtain an exclusive lock.
+		lock := syscall.Flock_t{Type: lockType}
+		err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock)
 		if err == nil {
 			return nil
 		} else if err != syscall.EAGAIN {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 

backend/vendor/github.com/coreos/bbolt/bolt_windows.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -16,8 +16,6 @@ var (
 )
 
 const (
-	lockExt = ".lock"
-
 	// see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
 	flagLockExclusive       = 2
 	flagLockFailImmediately = 1
@@ -48,48 +46,47 @@ func fdatasync(db *DB) error {
 }
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
-	// Create a separate lock file on windows because a process
-	// cannot share an exclusive lock on the same file. This is
-	// needed during Tx.WriteTo().
-	f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode)
-	if err != nil {
-		return err
-	}
-	db.lockfile = f
-
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	var flag uint32 = flagLockFailImmediately
+	if exclusive {
+		flag |= flagLockExclusive
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
+		// Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range
+		// -1..0 as the lock on the database file.
+		var m1 uint32 = (1 << 32) - 1 // -1 in a uint32
+		err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{
+			Offset:     m1,
+			OffsetHigh: m1,
+		})
 
-		var flag uint32 = flagLockFailImmediately
-		if exclusive {
-			flag |= flagLockExclusive
-		}
-
-		err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{})
 		if err == nil {
 			return nil
 		} else if err != errLockViolation {
 			return err
 		}
 
+		// If we timed oumercit then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
 // funlock releases an advisory lock on a file descriptor.
 func funlock(db *DB) error {
-	err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{})
-	db.lockfile.Close()
-	os.Remove(db.path+lockExt)
+	var m1 uint32 = (1 << 32) - 1 // -1 in a uint32
+	err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{
+		Offset:     m1,
+		OffsetHigh: m1,
+	})
 	return err
 }
 

backend/vendor/github.com/coreos/bbolt/boltsync_unix.go

@@ -1,6 +1,6 @@
 // +build !windows,!plan9,!linux,!openbsd
 
-package bolt
+package bbolt
 
 // fdatasync flushes written data to a file descriptor.
 func fdatasync(db *DB) error {

backend/vendor/github.com/coreos/bbolt/bucket.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"bytes"
@@ -14,13 +14,6 @@ const (
 	MaxValueSize = (1 << 31) - 2
 )
 
-const (
-	maxUint = ^uint(0)
-	minUint = 0
-	maxInt  = int(^uint(0) >> 1)
-	minInt  = -maxInt - 1
-)
-
 const bucketHeaderSize = int(unsafe.Sizeof(bucket{}))
 
 const (
@@ -130,9 +123,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket {
 func (b *Bucket) openBucket(value []byte) *Bucket {
 	var child = newBucket(b.tx)
 
+	// If unaligned load/stores are broken on this arch and value is
+	// unaligned simply clone to an aligned byte array.
+	unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0
+
+	if unaligned {
+		value = cloneBytes(value)
+	}
+
 	// If this is a writable transaction then we need to copy the bucket entry.
 	// Read-only transactions can point directly at the mmap entry.
-	if b.tx.writable {
+	if b.tx.writable && !unaligned {
 		child.bucket = &bucket{}
 		*child.bucket = *(*bucket)(unsafe.Pointer(&value[0]))
 	} else {
@@ -167,9 +168,8 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) {
 	if bytes.Equal(key, k) {
 		if (flags & bucketLeafFlag) != 0 {
 			return nil, ErrBucketExists
-		} else {
-			return nil, ErrIncompatibleValue
 		}
+		return nil, ErrIncompatibleValue
 	}
 
 	// Create empty, inline bucket.
@@ -316,7 +316,12 @@ func (b *Bucket) Delete(key []byte) error {
 
 	// Move cursor to correct position.
 	c := b.Cursor()
-	_, _, flags := c.seek(key)
+	k, _, flags := c.seek(key)
+
+	// Return nil if the key doesn't exist.
+	if !bytes.Equal(key, k) {
+		return nil
+	}
 
 	// Return an error if there is already existing bucket value.
 	if (flags & bucketLeafFlag) != 0 {
@@ -329,6 +334,28 @@ func (b *Bucket) Delete(key []byte) error {
 	return nil
 }
 
+// Sequence returns the current integer for the bucket without incrementing it.
+func (b *Bucket) Sequence() uint64 { return b.bucket.sequence }
+
+// SetSequence updates the sequence number for the bucket.
+func (b *Bucket) SetSequence(v uint64) error {
+	if b.tx.db == nil {
+		return ErrTxClosed
+	} else if !b.Writable() {
+		return ErrTxNotWritable
+	}
+
+	// Materialize the root node if it hasn't been already so that the
+	// bucket will be saved during commit.
+	if b.rootNode == nil {
+		_ = b.node(b.root, nil)
+	}
+
+	// Increment and return the sequence.
+	b.bucket.sequence = v
+	return nil
+}
+
 // NextSequence returns an autoincrementing integer for the bucket.
 func (b *Bucket) NextSequence() (uint64, error) {
 	if b.tx.db == nil {

backend/vendor/github.com/coreos/bbolt/cursor.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"bytes"
@@ -157,12 +157,6 @@ func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) {
 	// Start from root page/node and traverse to correct page.
 	c.stack = c.stack[:0]
 	c.search(seek, c.bucket.root)
-	ref := &c.stack[len(c.stack)-1]
-
-	// If the cursor is pointing to the end of page/node then return nil.
-	if ref.index >= ref.count() {
-		return nil, nil, 0
-	}
 
 	// If this is a bucket then return a nil value.
 	return c.keyValue()
@@ -339,6 +333,8 @@ func (c *Cursor) nsearch(key []byte) {
 // keyValue returns the key and value of the current leaf element.
 func (c *Cursor) keyValue() ([]byte, []byte, uint32) {
 	ref := &c.stack[len(c.stack)-1]
+
+	// If the cursor is pointing to the end of page/node then return nil.
 	if ref.count() == 0 || ref.index >= ref.count() {
 		return nil, nil, 0
 	}

backend/vendor/github.com/coreos/bbolt/db.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"errors"
@@ -7,8 +7,7 @@ import (
 	"log"
 	"os"
 	"runtime"
-	"runtime/debug"
-	"strings"
+	"sort"
 	"sync"
 	"time"
 	"unsafe"
@@ -23,6 +22,8 @@ const version = 2
 // Represents a marker value to indicate that a file is a Bolt DB.
 const magic uint32 = 0xED0CDAED
 
+const pgidNoFreelist pgid = 0xffffffffffffffff
+
 // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when
 // syncing changes to a file.  This is required as some operating systems,
 // such as OpenBSD, do not have a unified buffer cache (UBC) and writes
@@ -39,6 +40,19 @@ const (
 // default page size for db is set to the OS page size.
 var defaultPageSize = os.Getpagesize()
 
+// The time elapsed between consecutive file locking attempts.
+const flockRetryTimeout = 50 * time.Millisecond
+
+// FreelistType is the type of the freelist backend
+type FreelistType string
+
+const (
+	// FreelistArrayType indicates backend freelist type is array
+	FreelistArrayType = FreelistType("array")
+	// FreelistMapType indicates backend freelist type is hashmap
+	FreelistMapType = FreelistType("hashmap")
+)
+
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -61,6 +75,18 @@ type DB struct {
 	// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
 	NoSync bool
 
+	// When true, skips syncing freelist to disk. This improves the database
+	// write performance under normal operation, but requires a full database
+	// re-sync during recovery.
+	NoFreelistSync bool
+
+	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
+	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// The alternative one is using hashmap, it is faster in almost all circumstances
+	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
+	// The default type is array
+	FreelistType FreelistType
+
 	// When true, skips the truncate call when growing the database.
 	// Setting this to true is only safe on non-ext3/ext4 systems.
 	// Skipping truncation avoids preallocation of hard drive space and
@@ -96,8 +122,7 @@ type DB struct {
 
 	path     string
 	file     *os.File
-	lockfile *os.File // windows only
-	dataref  []byte   // mmap'ed readonly, write throws SEGV
+	dataref  []byte // mmap'ed readonly, write throws SEGV
 	data     *[maxMapSize]byte
 	datasz   int
 	filesz   int // current on disk file size
@@ -107,9 +132,11 @@ type DB struct {
 	opened   bool
 	rwtx     *Tx
 	txs      []*Tx
-	freelist *freelist
 	stats    Stats
 
+	freelist     *freelist
+	freelistLoad sync.Once
+
 	pagePool sync.Pool
 
 	batchMu sync.Mutex
@@ -148,14 +175,18 @@ func (db *DB) String() string {
 // If the file does not exist then it will be created automatically.
 // Passing in nil options will cause Bolt to open the database with the default options.
 func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
-	var db = &DB{opened: true}
-
+	db := &DB{
+		opened: true,
+	}
 	// Set default options if no options are provided.
 	if options == nil {
 		options = DefaultOptions
 	}
+	db.NoSync = options.NoSync
 	db.NoGrowSync = options.NoGrowSync
 	db.MmapFlags = options.MmapFlags
+	db.NoFreelistSync = options.NoFreelistSync
+	db.FreelistType = options.FreelistType
 
 	// Set default values for later DB operations.
 	db.MaxBatchSize = DefaultMaxBatchSize
@@ -183,7 +214,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// if !options.ReadOnly.
 	// The database file is locked using the shared lock (more than one process may
 	// hold a lock at the same time) otherwise (options.ReadOnly is set).
-	if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil {
+	if err := flock(db, !db.readOnly, options.Timeout); err != nil {
 		_ = db.close()
 		return nil, err
 	}
@@ -191,31 +222,41 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// Default values for test hooks
 	db.ops.writeAt = db.file.WriteAt
 
+	if db.pageSize = options.PageSize; db.pageSize == 0 {
+		// Set the default page size to the OS page size.
+		db.pageSize = defaultPageSize
+	}
+
 	// Initialize the database if it doesn't exist.
 	if info, err := db.file.Stat(); err != nil {
+		_ = db.close()
 		return nil, err
 	} else if info.Size() == 0 {
 		// Initialize new files with meta pages.
 		if err := db.init(); err != nil {
+			// clean up file descriptor on initialization fail
+			_ = db.close()
 			return nil, err
 		}
 	} else {
 		// Read the first meta page to determine the page size.
 		var buf [0x1000]byte
-		if _, err := db.file.ReadAt(buf[:], 0); err == nil {
-			m := db.pageInBuffer(buf[:], 0).meta()
-			if err := m.validate(); err != nil {
-				// If we can't read the page size, we can assume it's the same
-				// as the OS -- since that's how the page size was chosen in the
-				// first place.
-				//
-				// If the first page is invalid and this OS uses a different
-				// page size than what the database was created with then we
-				// are out of luck and cannot access the database.
-				db.pageSize = os.Getpagesize()
-			} else {
+		// If we can't read the page size, but can read a page, assume
+		// it's the same as the OS or one given -- since that's how the
+		// page size was chosen in the first place.
+		//
+		// If the first page is invalid and this OS uses a different
+		// page size than what the database was created with then we
+		// are out of luck and cannot access the database.
+		//
+		// TODO: scan for next page
+		if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
+			if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
 				db.pageSize = int(m.pageSize)
 			}
+		} else {
+			_ = db.close()
+			return nil, ErrInvalid
 		}
 	}
 
@@ -232,14 +273,50 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
-	// Read in the freelist.
-	db.freelist = newFreelist()
-	db.freelist.read(db.page(db.meta().freelist))
+	if db.readOnly {
+		return db, nil
+	}
+
+	db.loadFreelist()
+
+	// Flush freelist when transitioning from no sync to sync so
+	// NoFreelistSync unaware boltdb can open the db later.
+	if !db.NoFreelistSync && !db.hasSyncedFreelist() {
+		tx, err := db.Begin(true)
+		if tx != nil {
+			err = tx.Commit()
+		}
+		if err != nil {
+			_ = db.close()
+			return nil, err
+		}
+	}
 
 	// Mark the database as opened and return.
 	return db, nil
 }
 
+// loadFreelist reads the freelist if it is synced, or reconstructs it
+// by scanning the DB if it is not synced. It assumes there are no
+// concurrent accesses being made to the freelist.
+func (db *DB) loadFreelist() {
+	db.freelistLoad.Do(func() {
+		db.freelist = newFreelist(db.FreelistType)
+		if !db.hasSyncedFreelist() {
+			// Reconstruct free list by scanning the DB.
+			db.freelist.readIDs(db.freepages())
+		} else {
+			// Read free list from freelist page.
+			db.freelist.read(db.page(db.meta().freelist))
+		}
+		db.stats.FreePageN = db.freelist.free_count()
+	})
+}
+
+func (db *DB) hasSyncedFreelist() bool {
+	return db.meta().freelist != pgidNoFreelist
+}
+
 // mmap opens the underlying memory-mapped file and initializes the meta references.
 // minsz is the minimum size that the new mmap can be.
 func (db *DB) mmap(minsz int) error {
@@ -341,9 +418,6 @@ func (db *DB) mmapSize(size int) (int, error) {
 
 // init creates a new database file and initializes its meta pages.
 func (db *DB) init() error {
-	// Set the page size to the OS page size.
-	db.pageSize = os.Getpagesize()
-
 	// Create two meta pages on a buffer.
 	buf := make([]byte, db.pageSize*4)
 	for i := 0; i < 2; i++ {
@@ -387,7 +461,8 @@ func (db *DB) init() error {
 }
 
 // Close releases all database resources.
-// All transactions must be closed before closing the database.
+// It will block waiting for any open transactions to finish
+// before closing the database and returning.
 func (db *DB) Close() error {
 	db.rwlock.Lock()
 	defer db.rwlock.Unlock()
@@ -395,8 +470,8 @@ func (db *DB) Close() error {
 	db.metalock.Lock()
 	defer db.metalock.Unlock()
 
-	db.mmaplock.RLock()
-	defer db.mmaplock.RUnlock()
+	db.mmaplock.Lock()
+	defer db.mmaplock.Unlock()
 
 	return db.close()
 }
@@ -526,21 +601,36 @@ func (db *DB) beginRWTx() (*Tx, error) {
 	t := &Tx{writable: true}
 	t.init(db)
 	db.rwtx = t
+	db.freePages()
+	return t, nil
+}
 
-	// Free any pages associated with closed read-only transactions.
-	var minid txid = 0xFFFFFFFFFFFFFFFF
-	for _, t := range db.txs {
-		if t.meta.txid < minid {
-			minid = t.meta.txid
-		}
+// freePages releases any pages associated with closed read-only transactions.
+func (db *DB) freePages() {
+	// Free all pending pages prior to earliest open transaction.
+	sort.Sort(txsById(db.txs))
+	minid := txid(0xFFFFFFFFFFFFFFFF)
+	if len(db.txs) > 0 {
+		minid = db.txs[0].meta.txid
 	}
 	if minid > 0 {
 		db.freelist.release(minid - 1)
 	}
-
-	return t, nil
+	// Release unused txid extents.
+	for _, t := range db.txs {
+		db.freelist.releaseRange(minid, t.meta.txid-1)
+		minid = t.meta.txid + 1
+	}
+	db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF))
+	// Any page both allocated and freed in an extent is safe to release.
 }
 
+type txsById []*Tx
+
+func (t txsById) Len() int           { return len(t) }
+func (t txsById) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
+func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid }
+
 // removeTx removes a transaction from the database.
 func (db *DB) removeTx(tx *Tx) {
 	// Release the read lock on the mmap.
@@ -552,7 +642,10 @@ func (db *DB) removeTx(tx *Tx) {
 	// Remove the transaction.
 	for i, t := range db.txs {
 		if t == tx {
-			db.txs = append(db.txs[:i], db.txs[i+1:]...)
+			last := len(db.txs) - 1
+			db.txs[i] = db.txs[last]
+			db.txs[last] = nil
+			db.txs = db.txs[:last]
 			break
 		}
 	}
@@ -630,11 +723,7 @@ func (db *DB) View(fn func(*Tx) error) error {
 		return err
 	}
 
-	if err := t.Rollback(); err != nil {
-		return err
-	}
-
-	return nil
+	return t.Rollback()
 }
 
 // Batch calls fn as part of a batch. It behaves similar to Update,
@@ -734,9 +823,7 @@ retry:
 
 		// pass success, or bolt internal errors, to all callers
 		for _, c := range b.calls {
-			if c.err != nil {
-				c.err <- err
-			}
+			c.err <- err
 		}
 		break retry
 	}
@@ -823,7 +910,7 @@ func (db *DB) meta() *meta {
 }
 
 // allocate returns a contiguous block of memory starting at a given page.
-func (db *DB) allocate(count int) (*page, error) {
+func (db *DB) allocate(txid txid, count int) (*page, error) {
 	// Allocate a temporary buffer for the page.
 	var buf []byte
 	if count == 1 {
@@ -835,7 +922,7 @@ func (db *DB) allocate(count int) (*page, error) {
 	p.overflow = uint32(count - 1)
 
 	// Use pages from the freelist if they are available.
-	if p.id = db.freelist.allocate(count); p.id != 0 {
+	if p.id = db.freelist.allocate(txid, count); p.id != 0 {
 		return p, nil
 	}
 
@@ -890,6 +977,38 @@ func (db *DB) IsReadOnly() bool {
 	return db.readOnly
 }
 
+func (db *DB) freepages() []pgid {
+	tx, err := db.beginTx()
+	defer func() {
+		err = tx.Rollback()
+		if err != nil {
+			panic("freepages: failed to rollback tx")
+		}
+	}()
+	if err != nil {
+		panic("freepages: failed to open read only tx")
+	}
+
+	reachable := make(map[pgid]*page)
+	nofreed := make(map[pgid]bool)
+	ech := make(chan error)
+	go func() {
+		for e := range ech {
+			panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e))
+		}
+	}()
+	tx.checkBucket(&tx.root, reachable, nofreed, ech)
+	close(ech)
+
+	var fids []pgid
+	for i := pgid(2); i < db.meta().pgid; i++ {
+		if _, ok := reachable[i]; !ok {
+			fids = append(fids, i)
+		}
+	}
+	return fids
+}
+
 // Options represents the options that can be set when opening a database.
 type Options struct {
 	// Timeout is the amount of time to wait to obtain a file lock.
@@ -900,6 +1019,17 @@ type Options struct {
 	// Sets the DB.NoGrowSync flag before memory mapping the file.
 	NoGrowSync bool
 
+	// Do not sync freelist to disk. This improves the database write performance
+	// under normal operation, but requires a full database re-sync during recovery.
+	NoFreelistSync bool
+
+	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
+	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// The alternative one is using hashmap, it is faster in almost all circumstances
+	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
+	// The default type is array
+	FreelistType FreelistType
+
 	// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
 	// grab a shared lock (UNIX).
 	ReadOnly bool
@@ -916,13 +1046,22 @@ type Options struct {
 	// If initialMmapSize is smaller than the previous database size,
 	// it takes no effect.
 	InitialMmapSize int
+
+	// PageSize overrides the default OS page size.
+	PageSize int
+
+	// NoSync sets the initial value of DB.NoSync. Normally this can just be
+	// set directly on the DB itself when returned from Open(), but this option
+	// is useful in APIs which expose Options but not the underlying DB.
+	NoSync bool
 }
 
 // DefaultOptions represent the options used if nil options are passed into Open().
 // No timeout is used which will cause Bolt to wait indefinitely for a lock.
 var DefaultOptions = &Options{
-	Timeout:    0,
-	NoGrowSync: false,
+	Timeout:      0,
+	NoGrowSync:   false,
+	FreelistType: FreelistArrayType,
 }
 
 // Stats represents statistics about the database.
@@ -952,15 +1091,11 @@ func (s *Stats) Sub(other *Stats) Stats {
 	diff.PendingPageN = s.PendingPageN
 	diff.FreeAlloc = s.FreeAlloc
 	diff.FreelistInuse = s.FreelistInuse
-	diff.TxN = other.TxN - s.TxN
+	diff.TxN = s.TxN - other.TxN
 	diff.TxStats = s.TxStats.Sub(&other.TxStats)
 	return diff
 }
 
-func (s *Stats) add(other *Stats) {
-	s.TxStats.add(&other.TxStats)
-}
-
 type Info struct {
 	Data     uintptr
 	PageSize int
@@ -999,7 +1134,8 @@ func (m *meta) copy(dest *meta) {
 func (m *meta) write(p *page) {
 	if m.root.root >= m.pgid {
 		panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid))
-	} else if m.freelist >= m.pgid {
+	} else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist {
+		// TODO: reject pgidNoFreeList if !NoFreelistSync
 		panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid))
 	}
 
@@ -1026,11 +1162,3 @@ func _assert(condition bool, msg string, v ...interface{}) {
 		panic(fmt.Sprintf("assertion failed: "+msg, v...))
 	}
 }
-
-func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
-func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
-
-func printstack() {
-	stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n")
-	fmt.Fprintln(os.Stderr, stack)
-}

backend/vendor/github.com/coreos/bbolt/doc.go

@@ -1,5 +1,5 @@
 /*
-Package bolt implements a low-level key/value store in pure Go. It supports
+package bbolt implements a low-level key/value store in pure Go. It supports
 fully serializable transactions, ACID semantics, and lock-free MVCC with
 multiple readers and a single writer. Bolt can be used for projects that
 want a simple data store without the need to add large dependencies such as
@@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic.
 
 
 */
-package bolt
+package bbolt

backend/vendor/github.com/coreos/bbolt/errors.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import "errors"
 

backend/vendor/github.com/coreos/bbolt/freelist.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -6,25 +6,72 @@ import (
 	"unsafe"
 )
 
+// txPending holds a list of pgids and corresponding allocation txns
+// that are pending to be freed.
+type txPending struct {
+	ids              []pgid
+	alloctx          []txid // txids allocating the ids
+	lastReleaseBegin txid   // beginning txid of last matching releaseRange
+}
+
+// pidSet holds the set of starting pgids which have the same span size
+type pidSet map[pgid]struct{}
+
 // freelist represents a list of all pages that are available for allocation.
 // It also tracks pages that have been freed but are still in use by open transactions.
 type freelist struct {
-	ids     []pgid          // all free and available free page ids.
-	pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
-	cache   map[pgid]bool   // fast lookup of all free and pending page ids.
+	freelistType   FreelistType                // freelist type
+	ids            []pgid                      // all free and available free page ids.
+	allocs         map[pgid]txid               // mapping of txid that allocated a pgid.
+	pending        map[txid]*txPending         // mapping of soon-to-be free page ids by tx.
+	cache          map[pgid]bool               // fast lookup of all free and pending page ids.
+	freemaps       map[uint64]pidSet           // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size
+	forwardMap     map[pgid]uint64             // key is start pgid, value is its span size
+	backwardMap    map[pgid]uint64             // key is end pgid, value is its span size
+	allocate       func(txid txid, n int) pgid // the freelist allocate func
+	free_count     func() int                  // the function which gives you free page number
+	mergeSpans     func(ids pgids)             // the mergeSpan func
+	getFreePageIDs func() []pgid               // get free pgids func
+	readIDs        func(pgids []pgid)          // readIDs func reads list of pages and init the freelist
 }
 
 // newFreelist returns an empty, initialized freelist.
-func newFreelist() *freelist {
-	return &freelist{
-		pending: make(map[txid][]pgid),
-		cache:   make(map[pgid]bool),
+func newFreelist(freelistType FreelistType) *freelist {
+	f := &freelist{
+		freelistType: freelistType,
+		allocs:       make(map[pgid]txid),
+		pending:      make(map[txid]*txPending),
+		cache:        make(map[pgid]bool),
+		freemaps:     make(map[uint64]pidSet),
+		forwardMap:   make(map[pgid]uint64),
+		backwardMap:  make(map[pgid]uint64),
 	}
+
+	if freelistType == FreelistMapType {
+		f.allocate = f.hashmapAllocate
+		f.free_count = f.hashmapFreeCount
+		f.mergeSpans = f.hashmapMergeSpans
+		f.getFreePageIDs = f.hashmapGetFreePageIDs
+		f.readIDs = f.hashmapReadIDs
+	} else {
+		f.allocate = f.arrayAllocate
+		f.free_count = f.arrayFreeCount
+		f.mergeSpans = f.arrayMergeSpans
+		f.getFreePageIDs = f.arrayGetFreePageIDs
+		f.readIDs = f.arrayReadIDs
+	}
+
+	return f
 }
 
 // size returns the size of the page after serialization.
 func (f *freelist) size() int {
-	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count())
+	n := f.count()
+	if n >= 0xFFFF {
+		// The first element will be used to store the count. See freelist.write.
+		n++
+	}
+	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n)
 }
 
 // count returns count of pages on the freelist
@@ -32,35 +79,34 @@ func (f *freelist) count() int {
 	return f.free_count() + f.pending_count()
 }
 
-// free_count returns count of free pages
-func (f *freelist) free_count() int {
+// arrayFreeCount returns count of free pages(array version)
+func (f *freelist) arrayFreeCount() int {
 	return len(f.ids)
 }
 
 // pending_count returns count of pending pages
 func (f *freelist) pending_count() int {
 	var count int
-	for _, list := range f.pending {
-		count += len(list)
+	for _, txp := range f.pending {
+		count += len(txp.ids)
 	}
 	return count
 }
 
-// all returns a list of all free ids and all pending ids in one sorted list.
-func (f *freelist) all() []pgid {
-	m := make(pgids, 0)
-
-	for _, list := range f.pending {
-		m = append(m, list...)
+// copyall copies into dst a list of all free ids and all pending ids in one sorted list.
+// f.count returns the minimum length required for dst.
+func (f *freelist) copyall(dst []pgid) {
+	m := make(pgids, 0, f.pending_count())
+	for _, txp := range f.pending {
+		m = append(m, txp.ids...)
 	}
-
 	sort.Sort(m)
-	return pgids(f.ids).merge(m)
+	mergepgids(dst, f.getFreePageIDs(), m)
 }
 
-// allocate returns the starting page id of a contiguous list of pages of a given size.
+// arrayAllocate returns the starting page id of a contiguous list of pages of a given size.
 // If a contiguous block cannot be found then 0 is returned.
-func (f *freelist) allocate(n int) pgid {
+func (f *freelist) arrayAllocate(txid txid, n int) pgid {
 	if len(f.ids) == 0 {
 		return 0
 	}
@@ -93,7 +139,7 @@ func (f *freelist) allocate(n int) pgid {
 			for i := pgid(0); i < pgid(n); i++ {
 				delete(f.cache, initial+i)
 			}
-
+			f.allocs[initial] = txid
 			return initial
 		}
 
@@ -110,44 +156,103 @@ func (f *freelist) free(txid txid, p *page) {
 	}
 
 	// Free page and all its overflow pages.
-	var ids = f.pending[txid]
+	txp := f.pending[txid]
+	if txp == nil {
+		txp = &txPending{}
+		f.pending[txid] = txp
+	}
+	allocTxid, ok := f.allocs[p.id]
+	if ok {
+		delete(f.allocs, p.id)
+	} else if (p.flags & freelistPageFlag) != 0 {
+		// Freelist is always allocated by prior tx.
+		allocTxid = txid - 1
+	}
+
 	for id := p.id; id <= p.id+pgid(p.overflow); id++ {
 		// Verify that page is not already free.
 		if f.cache[id] {
 			panic(fmt.Sprintf("page %d already freed", id))
 		}
-
 		// Add to the freelist and cache.
-		ids = append(ids, id)
+		txp.ids = append(txp.ids, id)
+		txp.alloctx = append(txp.alloctx, allocTxid)
 		f.cache[id] = true
 	}
-	f.pending[txid] = ids
 }
 
 // release moves all page ids for a transaction id (or older) to the freelist.
 func (f *freelist) release(txid txid) {
 	m := make(pgids, 0)
-	for tid, ids := range f.pending {
+	for tid, txp := range f.pending {
 		if tid <= txid {
 			// Move transaction's pending pages to the available freelist.
 			// Don't remove from the cache since the page is still free.
-			m = append(m, ids...)
+			m = append(m, txp.ids...)
 			delete(f.pending, tid)
 		}
 	}
-	sort.Sort(m)
-	f.ids = pgids(f.ids).merge(m)
+	f.mergeSpans(m)
+}
+
+// releaseRange moves pending pages allocated within an extent [begin,end] to the free list.
+func (f *freelist) releaseRange(begin, end txid) {
+	if begin > end {
+		return
+	}
+	var m pgids
+	for tid, txp := range f.pending {
+		if tid < begin || tid > end {
+			continue
+		}
+		// Don't recompute freed pages if ranges haven't updated.
+		if txp.lastReleaseBegin == begin {
+			continue
+		}
+		for i := 0; i < len(txp.ids); i++ {
+			if atx := txp.alloctx[i]; atx < begin || atx > end {
+				continue
+			}
+			m = append(m, txp.ids[i])
+			txp.ids[i] = txp.ids[len(txp.ids)-1]
+			txp.ids = txp.ids[:len(txp.ids)-1]
+			txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1]
+			txp.alloctx = txp.alloctx[:len(txp.alloctx)-1]
+			i--
+		}
+		txp.lastReleaseBegin = begin
+		if len(txp.ids) == 0 {
+			delete(f.pending, tid)
+		}
+	}
+	f.mergeSpans(m)
 }
 
 // rollback removes the pages from a given pending tx.
 func (f *freelist) rollback(txid txid) {
 	// Remove page ids from cache.
-	for _, id := range f.pending[txid] {
-		delete(f.cache, id)
+	txp := f.pending[txid]
+	if txp == nil {
+		return
 	}
-
-	// Remove pages from pending list.
+	var m pgids
+	for i, pgid := range txp.ids {
+		delete(f.cache, pgid)
+		tx := txp.alloctx[i]
+		if tx == 0 {
+			continue
+		}
+		if tx != txid {
+			// Pending free aborted; restore page back to alloc list.
+			f.allocs[pgid] = tx
+		} else {
+			// Freed page was allocated by this txn; OK to throw away.
+			m = append(m, pgid)
+		}
+	}
+	// Remove pages from pending list and mark as free if allocated by txid.
 	delete(f.pending, txid)
+	f.mergeSpans(m)
 }
 
 // freed returns whether a given page is in the free list.
@@ -157,6 +262,9 @@ func (f *freelist) freed(pgid pgid) bool {
 
 // read initializes the freelist from a freelist page.
 func (f *freelist) read(p *page) {
+	if (p.flags & freelistPageFlag) == 0 {
+		panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ()))
+	}
 	// If the page.count is at the max uint16 value (64k) then it's considered
 	// an overflow and the size of the freelist is stored as the first element.
 	idx, count := 0, int(p.count)
@@ -169,39 +277,49 @@ func (f *freelist) read(p *page) {
 	if count == 0 {
 		f.ids = nil
 	} else {
-		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count]
-		f.ids = make([]pgid, len(ids))
-		copy(f.ids, ids)
+		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count]
 
+		// copy the ids, so we don't modify on the freelist page directly
+		idsCopy := make([]pgid, count)
+		copy(idsCopy, ids)
 		// Make sure they're sorted.
-		sort.Sort(pgids(f.ids))
-	}
+		sort.Sort(pgids(idsCopy))
 
-	// Rebuild the page cache.
+		f.readIDs(idsCopy)
+	}
+}
+
+// arrayReadIDs initializes the freelist from a given list of ids.
+func (f *freelist) arrayReadIDs(ids []pgid) {
+	f.ids = ids
 	f.reindex()
 }
 
+func (f *freelist) arrayGetFreePageIDs() []pgid {
+	return f.ids
+}
+
 // write writes the page ids onto a freelist page. All free and pending ids are
 // saved to disk since in the event of a program crash, all pending ids will
 // become free.
 func (f *freelist) write(p *page) error {
 	// Combine the old free pgids and pgids waiting on an open transaction.
-	ids := f.all()
 
 	// Update the header flag.
 	p.flags |= freelistPageFlag
 
 	// The page.count can only hold up to 64k elements so if we overflow that
 	// number then we handle it by putting the size in the first element.
-	if len(ids) == 0 {
-		p.count = uint16(len(ids))
-	} else if len(ids) < 0xFFFF {
-		p.count = uint16(len(ids))
-		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids)
+	lenids := f.count()
+	if lenids == 0 {
+		p.count = uint16(lenids)
+	} else if lenids < 0xFFFF {
+		p.count = uint16(lenids)
+		f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:])
 	} else {
 		p.count = 0xFFFF
-		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids))
-		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids)
+		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids)
+		f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:])
 	}
 
 	return nil
@@ -213,8 +331,8 @@ func (f *freelist) reload(p *page) {
 
 	// Build a cache of only pending pages.
 	pcache := make(map[pgid]bool)
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			pcache[pendingID] = true
 		}
 	}
@@ -222,27 +340,31 @@ func (f *freelist) reload(p *page) {
 	// Check each page in the freelist and build a new available freelist
 	// with any pages not in the pending lists.
 	var a []pgid
-	for _, id := range f.ids {
+	for _, id := range f.getFreePageIDs() {
 		if !pcache[id] {
 			a = append(a, id)
 		}
 	}
-	f.ids = a
 
-	// Once the available list is rebuilt then rebuild the free cache so that
-	// it includes the available and pending free pages.
-	f.reindex()
+	f.readIDs(a)
 }
 
 // reindex rebuilds the free cache based on available and pending free lists.
 func (f *freelist) reindex() {
-	f.cache = make(map[pgid]bool)
-	for _, id := range f.ids {
+	ids := f.getFreePageIDs()
+	f.cache = make(map[pgid]bool, len(ids))
+	for _, id := range ids {
 		f.cache[id] = true
 	}
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			f.cache[pendingID] = true
 		}
 	}
 }
+
+// arrayMergeSpans try to merge list of pages(represented by pgids) with existing spans but using array
+func (f *freelist) arrayMergeSpans(ids pgids) {
+	sort.Sort(ids)
+	f.ids = pgids(f.ids).merge(ids)
+}

backend/vendor/github.com/coreos/bbolt/freelist_hmap.go

@@ -0,0 +1,178 @@
+package bbolt
+
+import "sort"
+
+// hashmapFreeCount returns count of free pages(hashmap version)
+func (f *freelist) hashmapFreeCount() int {
+	// use the forwardmap to get the total count
+	count := 0
+	for _, size := range f.forwardMap {
+		count += int(size)
+	}
+	return count
+}
+
+// hashmapAllocate serves the same purpose as arrayAllocate, but use hashmap as backend
+func (f *freelist) hashmapAllocate(txid txid, n int) pgid {
+	if n == 0 {
+		return 0
+	}
+
+	// if we have a exact size match just return short path
+	if bm, ok := f.freemaps[uint64(n)]; ok {
+		for pid := range bm {
+			// remove the span
+			f.delSpan(pid, uint64(n))
+
+			f.allocs[pid] = txid
+
+			for i := pgid(0); i < pgid(n); i++ {
+				delete(f.cache, pid+pgid(i))
+			}
+			return pid
+		}
+	}
+
+	// lookup the map to find larger span
+	for size, bm := range f.freemaps {
+		if size < uint64(n) {
+			continue
+		}
+
+		for pid := range bm {
+			// remove the initial
+			f.delSpan(pid, uint64(size))
+
+			f.allocs[pid] = txid
+
+			remain := size - uint64(n)
+
+			// add remain span
+			f.addSpan(pid+pgid(n), remain)
+
+			for i := pgid(0); i < pgid(n); i++ {
+				delete(f.cache, pid+pgid(i))
+			}
+			return pid
+		}
+	}
+
+	return 0
+}
+
+// hashmapReadIDs reads pgids as input an initial the freelist(hashmap version)
+func (f *freelist) hashmapReadIDs(pgids []pgid) {
+	f.init(pgids)
+
+	// Rebuild the page cache.
+	f.reindex()
+}
+
+// hashmapGetFreePageIDs returns the sorted free page ids
+func (f *freelist) hashmapGetFreePageIDs() []pgid {
+	count := f.free_count()
+	if count == 0 {
+		return nil
+	}
+
+	m := make([]pgid, 0, count)
+	for start, size := range f.forwardMap {
+		for i := 0; i < int(size); i++ {
+			m = append(m, start+pgid(i))
+		}
+	}
+	sort.Sort(pgids(m))
+
+	return m
+}
+
+// hashmapMergeSpans try to merge list of pages(represented by pgids) with existing spans
+func (f *freelist) hashmapMergeSpans(ids pgids) {
+	for _, id := range ids {
+		// try to see if we can merge and update
+		f.mergeWithExistingSpan(id)
+	}
+}
+
+// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward
+func (f *freelist) mergeWithExistingSpan(pid pgid) {
+	prev := pid - 1
+	next := pid + 1
+
+	preSize, mergeWithPrev := f.backwardMap[prev]
+	nextSize, mergeWithNext := f.forwardMap[next]
+	newStart := pid
+	newSize := uint64(1)
+
+	if mergeWithPrev {
+		//merge with previous span
+		start := prev + 1 - pgid(preSize)
+		f.delSpan(start, preSize)
+
+		newStart -= pgid(preSize)
+		newSize += preSize
+	}
+
+	if mergeWithNext {
+		// merge with next span
+		f.delSpan(next, nextSize)
+		newSize += nextSize
+	}
+
+	f.addSpan(newStart, newSize)
+}
+
+func (f *freelist) addSpan(start pgid, size uint64) {
+	f.backwardMap[start-1+pgid(size)] = size
+	f.forwardMap[start] = size
+	if _, ok := f.freemaps[size]; !ok {
+		f.freemaps[size] = make(map[pgid]struct{})
+	}
+
+	f.freemaps[size][start] = struct{}{}
+}
+
+func (f *freelist) delSpan(start pgid, size uint64) {
+	delete(f.forwardMap, start)
+	delete(f.backwardMap, start+pgid(size-1))
+	delete(f.freemaps[size], start)
+	if len(f.freemaps[size]) == 0 {
+		delete(f.freemaps, size)
+	}
+}
+
+// initial from pgids using when use hashmap version
+// pgids must be sorted
+func (f *freelist) init(pgids []pgid) {
+	if len(pgids) == 0 {
+		return
+	}
+
+	size := uint64(1)
+	start := pgids[0]
+
+	if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) {
+		panic("pgids not sorted")
+	}
+
+	f.freemaps = make(map[uint64]pidSet)
+	f.forwardMap = make(map[pgid]uint64)
+	f.backwardMap = make(map[pgid]uint64)
+
+	for i := 1; i < len(pgids); i++ {
+		// continuous page
+		if pgids[i] == pgids[i-1]+1 {
+			size++
+		} else {
+			f.addSpan(start, size)
+
+			size = 1
+			start = pgids[i]
+		}
+	}
+
+	// init the tail
+	if size != 0 && start != 0 {
+		f.addSpan(start, size)
+	}
+}