Merge pull request #317 from Reeywhaar/mv-web-to-frontend

Rename "web"  to "frontend"

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
+/frontend/node_modules/
-/web/public/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,5 +19,5 @@ debug
 debug.test
 *.prof
 *.test
-/bin/
 remark42
+/backend/var/

.drone.yml

@@ -5,7 +5,7 @@ workspace:
 pipeline:
   
   build:
-    image: golang:1.10-alpine
+    image: golang:1.12-alpine
     commands:
       - sleep 5
       - nslookup mongo
@@ -99,4 +99,3 @@ services:
   mongo:
     image: mongo:3.6
     command: [ --smallfiles ]
-

.gitignore

@@ -8,9 +8,9 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
+/frontend/node_modules/
-/web/public/
+/frontend/public/
-/web/coverage
+/frontend/coverage
 *.prof
 *.test
 /rest-client.env.json
@@ -18,3 +18,4 @@ debug.test
 .mongo
 remark42
 /bin/
+/backend/var/

Dockerfile

@@ -21,15 +21,17 @@ ARG DRONE_PULL_REQUEST
 ARG SKIP_BACKEND_TEST
 ARG MONGO_TEST
 
-WORKDIR /go/src/github.com/umputun/remark/backend
+ADD backend /build/backend
-ADD backend /go/src/github.com/umputun/remark/backend
+ADD .git /build/.git
-ADD .git /go/src/github.com/umputun/remark/.git
+WORKDIR /build/backend
 
 # run tests
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
 RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
@@ -37,39 +39,34 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
-# coverage report
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    mkdir -p target && /script/coverage.sh ; \
-    else echo "skip backend coverage" ; fi
-
 # submit coverage to coverals if COVERALLS_TOKEN in env
 RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
     echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
-    echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
-    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
+    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
+ADD frontend/package.json /srv/frontend/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
@@ -77,9 +74,9 @@ ARG CI
 ARG SKIP_FRONTEND_TEST
 ARG NODE_ENV=production
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD web /srv/web
+ADD frontend /srv/frontend
-RUN cd /srv/web && \
+RUN cd /srv/frontend && \
     if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
@@ -95,8 +92,8 @@ ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 

Dockerfile.artifacts

@@ -9,19 +9,20 @@ ARG DRONE_BRANCH
 ENV SKIP_FRONTEND_TEST=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
+ADD frontend/package.json /srv/frontend/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
 ARG CI
 ARG NODE_ENV=production
 ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD web /srv/web
+ADD frontend /srv/frontend
-RUN cd /srv/web && \
+RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
@@ -35,7 +36,7 @@ WORKDIR /go/src/github.com/umputun/remark/backend
 ADD backend /go/src/github.com/umputun/remark/backend
 ADD README.md /go/src/github.com/umputun/remark/
 ADD LICENSE /go/src/github.com/umputun/remark/
-COPY --from=build-frontend /srv/web/public/ web
+COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
     export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \

README.md

@@ -3,19 +3,22 @@
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
 * Social login via Google, Facebook, Github and Yandex
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
 * Import from disqus and wordpress
-* Markdown support
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
+* Telegram notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
 * Self-contained executable can be deployed directly to Linux, Windows and MacOS
-* Clean, lightweight and fully customizable UI
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
@@ -29,6 +32,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
       - [Without docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
+      - [Quick installation test](#quick-installation-test)
       - [Register oauth2 providers](#register-oauth2-providers)
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
@@ -70,7 +74,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
+* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
 * pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
@@ -83,60 +87,70 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
+| Command line            | Environment             | Default                  | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
-| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
-| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
-| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
-| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
-| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -170,6 +184,10 @@ services:
             - ./var:/srv/var                        # persistent volume to store all remark42 data
 ```
 
+#### Quick installation test
+
+To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
+
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
@@ -222,6 +240,13 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 
 For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
 
+##### Anonymous Auth Provider
+
+Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
+
+- name should be at least 3 characters long
+- name has to start from the letter and contains letters, numbers, underscores and spaces only.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
@@ -266,7 +291,7 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 
 ```
     environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
+        - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
 To get user id just login and click on your username or any other user you want to promote to admins.
@@ -425,9 +450,9 @@ To run backend with mongodb store mongo container should be started first - `doc
 
 * install [Node.js 8](https://nodejs.org/en/) or higher;
 * install [NPM 6.1.0](https://www.npmjs.com/package/npm);
-* run `npm install` inside `./web`;
+* run `npm install` inside `./frontend`;
 * run `npm run build` there;
-* result files will be saved in `./web/public`.
+* result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
 It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
@@ -448,6 +473,10 @@ npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
+#### Frontend guide
+
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
 ## API
 
 ### Authorization
@@ -481,8 +510,10 @@ type Comment struct {
     User      User            `json:"user"`    // user info, read only
     Locator   Locator         `json:"locator"` // post locator
     Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
+    Vote      int             `json:"vote"`    // vote for the current user, -1/1/0.
+    Controversy float64       `json:"controversy,omitempty"` // comment controversy, read only
     Timestamp time.Time       `json:"time"`    // time stamp, read only
+    Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
     PostTitle string          `json:"title"`   // post title
@@ -492,6 +523,11 @@ type Locator struct {
     SiteID string `json:"site"`     // site id
     URL    string `json:"url"`      // post url
 }
+
+type Edit struct {
+  Timestamp time.Time `json:"time" bson:"time"`
+  Summary   string    `json:"summary"`
+}
 ```
 
 * `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
@@ -518,11 +554,11 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
 ```go
- 	type EditRequest struct {
+   type EditRequest struct {
- 		Text    string `json:"text"`    // updated text
+     Text    string `json:"text"`    // updated text
- 		Summary string `json:"summary"` // optional, summary of the edit
+     Summary string `json:"summary"` // optional, summary of the edit
- 		Delete  bool   `json:"delete"`  // delete flag 
+     Delete  bool   `json:"delete"`  // delete flag
- 	}{}
+   }{}
 ```
 
 * `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
@@ -553,15 +589,21 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go
-  type config struct {
+  type Config struct {
-      Version       string   `json:"version"`
+      Version        string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
+      EditDuration   int      `json:"edit_duration"`
-      Admins        []string `json:"admins"`
+      MaxCommentSize int      `json:"max_comment_size"`
-      Auth          []string `json:"auth_providers"`
+      Admins         []string `json:"admins"`
-      LowScore      int      `json:"low_score"`
+      AdminEmail     string   `json:"admin_email"`
-      CriticalScore int      `json:"critical_score"`
+      Auth           []string `json:"auth_providers"`
+      LowScore       int      `json:"low_score"`
+      CriticalScore  int      `json:"critical_score"`
+      PositiveScore  bool     `json:"positive_score"`
+      ReadOnlyAge    int      `json:"readonly_age"`
+      MaxImageSize   int      `json:"max_image_size"`
   }
   ```
+
 * `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
 
 ### RSS feeds
@@ -570,6 +612,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.

backend/Gopkg.lock

@@ -1,429 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter",
-  ]
-  pruneopts = "UT"
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
-  name = "github.com/globalsign/mgo"
-  packages = [
-    ".",
-    "bson",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = "UT"
-  revision = "113d3961e7311526535a1ef7042196563d442761"
-  version = "r2018.06.15"
-
-[[projects]]
-  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware",
-  ]
-  pruneopts = "UT"
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
-  name = "github.com/go-chi/cors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dba6525398619dead495962a916728e7ee2ca322"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:d31c72099b9e195785933d11441bc92cce9b2c421f8fca712a7eaf36a8e55a47"
-  name = "github.com/go-pkgz/auth"
-  packages = [
-    ".",
-    "avatar",
-    "logger",
-    "middleware",
-    "provider",
-    "token",
-  ]
-  pruneopts = "UT"
-  revision = "92256b685196d26a346dfbedea12898fe589b0eb"
-  version = "v0.4.0"
-
-[[projects]]
-  digest = "1:1933dabfb0e07548ed9684fdef857cc6f2413bbbb74ba838a7c1f77407b8fd90"
-  name = "github.com/go-pkgz/lgr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8f56fccc1a15dfe9c7f72e049fc87cc378205b97"
-  version = "v0.2.2"
-
-[[projects]]
-  digest = "1:c509e3f646c48148f2239fa66d5cd62785dfd057b5d2b46972e45b60049c19ed"
-  name = "github.com/go-pkgz/mongo"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ee1e9a167094f8cdbc27407907aa694fff350ec3"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
-  name = "github.com/go-pkgz/repeater"
-  packages = [
-    ".",
-    "strategy",
-  ]
-  pruneopts = "UT"
-  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:9aba5c95373481f118e57e9740d9e82f86802d86849809ab89324bcb4f236451"
-  name = "github.com/go-pkgz/rest"
-  packages = [
-    ".",
-    "cache",
-    "logger",
-  ]
-  pruneopts = "UT"
-  revision = "e7d08d0194d613b8854de2e487bf7732500fa153"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = "UT"
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
-  name = "github.com/nullrocks/identicon"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7875f45b0022edded6377e40639d8aa620193a62"
-
-[[projects]]
-  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  pruneopts = "UT"
-  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
-  version = "v0.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
-  name = "golang.org/x/crypto"
-  packages = [
-    "acme",
-    "acme/autocert",
-  ]
-  pruneopts = "UT"
-  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64",
-  ]
-  pruneopts = "UT"
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom",
-  ]
-  pruneopts = "UT"
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex",
-  ]
-  pruneopts = "UT"
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = "UT"
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = "UT"
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/PuerkitoBio/goquery",
-    "github.com/coreos/bbolt",
-    "github.com/dgrijalva/jwt-go",
-    "github.com/didip/tollbooth",
-    "github.com/didip/tollbooth_chi",
-    "github.com/globalsign/mgo",
-    "github.com/globalsign/mgo/bson",
-    "github.com/go-chi/chi",
-    "github.com/go-chi/chi/middleware",
-    "github.com/go-chi/cors",
-    "github.com/go-chi/render",
-    "github.com/go-pkgz/auth",
-    "github.com/go-pkgz/auth/avatar",
-    "github.com/go-pkgz/auth/provider",
-    "github.com/go-pkgz/auth/token",
-    "github.com/go-pkgz/lgr",
-    "github.com/go-pkgz/mongo",
-    "github.com/go-pkgz/repeater",
-    "github.com/go-pkgz/rest",
-    "github.com/go-pkgz/rest/cache",
-    "github.com/go-pkgz/rest/logger",
-    "github.com/google/uuid",
-    "github.com/gorilla/feeds",
-    "github.com/hashicorp/go-multierror",
-    "github.com/jessevdk/go-flags",
-    "github.com/microcosm-cc/bluemonday",
-    "github.com/patrickmn/go-cache",
-    "github.com/pkg/errors",
-    "github.com/rakyll/statik/fs",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/acme/autocert",
-    "golang.org/x/net/html",
-    "gopkg.in/russross/blackfriday.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/Gopkg.toml

@@ -1,44 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-
-[[constraint]]
-  name = "github.com/globalsign/mgo"
-  version = "r2018.06.15"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-
-
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"

backend/app/cmd/avatar.go

@@ -31,6 +31,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL)
+	r, err := http.Get(listURL) //nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL)
+		r, err = http.Get(commentsURL) //nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}
@@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil

backend/app/cmd/cmd.go

@@ -33,7 +33,7 @@ type CommonOpts struct {
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
-	c.RemarkURL = commonOpts.RemarkURL
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
 	c.SharedSecret = commonOpts.SharedSecret
 	c.Revision = commonOpts.Revision
 }

backend/app/cmd/server.go

@@ -8,13 +8,14 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	auth_cache "github.com/patrickmn/go-cache"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
@@ -31,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -42,6 +44,7 @@ type ServerCommand struct {
 	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -53,23 +56,25 @@ type ServerCommand struct {
 	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
 	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
 	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
 	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
 	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
 	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
 	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" default:"" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
 
 	Auth struct {
 		TTL struct {
 			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
 			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
 		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Dev      bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
@@ -90,6 +95,22 @@ type StoreGroup struct {
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
 }
 
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
+}
+
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
@@ -159,6 +180,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -179,6 +201,7 @@ func (s *ServerCommand) Execute(args []string) error {
 	app, err := s.newServerApp()
 	if err != nil {
 		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
 		log.Printf("[ERROR] remark terminated with error %+v", err)
@@ -211,12 +234,19 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+
 	dataService := &service.DataStore{
 		Interface:              storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
+		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
@@ -272,15 +302,16 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     imageService,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
+		da, errDevAuth := authenticator.DevAuth()
-		if err != nil {
+		if errDevAuth != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -294,6 +325,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -319,12 +351,17 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -401,6 +438,27 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+				MaxHeight:  s.Image.ResizeHeight,
+				MaxWidth:   s.Image.ResizeWidth,
+			},
+			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
+		}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
@@ -471,10 +529,29 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 	if s.Auth.Dev {
+		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Anonymous {
+		log.Print("[INFO] anonymous access enabled")
+		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
+		authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
+			user = strings.TrimSpace(user)
+			if len(user) < 3 {
+				log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
+				return false, nil
+			}
+
+			if !isValidAnonName(user) {
+				log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
+				return false, nil
+			}
+			return true, nil
+		}))
+	}
+
 	if providers == 0 {
 		log.Printf("[WARN] no auth providers defined")
 	}
@@ -563,17 +640,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*auth_cache.Cache
+	*authcache.Cache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
 }
 
+// Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 	return c.Cache.Get(key.(string))
 }
 
+// Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
 }

backend/app/cmd/server_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})
@@ -83,6 +83,48 @@ func TestServerApp_DevMode(t *testing.T) {
 	app.Wait()
 }
 
+func TestServerApp_AnonMode(t *testing.T) {
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18085
+		o.Auth.Anonymous = true
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+
+	// send ping
+	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	// try to login with good name
+	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+
+	// try to login with bad name
+	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	// try to login with short name
+	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	app.Wait()
+}
 func TestServerApp_WithMongo(t *testing.T) {
 
 	mongoURL := os.Getenv("MONGO_TEST")
@@ -102,15 +144,15 @@ func TestServerApp_WithMongo(t *testing.T) {
 		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
 	require.Nil(t, err)
 	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation = "/tmp"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
 
 	// create app
 	app, err := opts.newServerApp()
 	require.Nil(t, err)
 
 	defer func() {
-		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
+		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, err)
+		assert.NoError(t, e)
 		conn := mongo.NewConnection(s, "test_remark", "")
 		_ = conn.WithDB(func(dbase *mgo.Database) error {
 			assert.NoError(t, dbase.DropDatabase())
@@ -145,8 +187,10 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
+		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
+		"--ssl.port=18443", "--image.fs.path=/tmp"})
 	require.Nil(t, err)
 
 	// create app
@@ -200,7 +244,7 @@ func TestServerApp_Failed(t *testing.T) {
 	p := flags.NewParser(&opts, flags.Default)
 
 	// RO bolt location
-	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
 	assert.Nil(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
@@ -264,7 +308,7 @@ func TestServerApp_MainSignal(t *testing.T) {
 
 	p := flags.NewParser(&s, flags.Default)
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
 	require.Nil(t, err)
@@ -316,7 +360,7 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})
@@ -345,7 +389,7 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
+	client := http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
@@ -390,7 +434,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
 	require.Nil(t, err)
-	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"

backend/app/main.go

@@ -62,7 +62,7 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 		return
 	}
 	log.Setup(log.Msec, log.LevelBraces)

backend/app/main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -11,20 +12,25 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(500 * time.Millisecond)
+		time.Sleep(5000 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.Nil(t, e)
 	}()
 
 	wg := sync.WaitGroup{}
@@ -32,20 +38,29 @@ func TestMain(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
 		wg.Done()
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	var passed bool
+	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
+		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		if e != nil {
+			t.Logf("%+v", e)
+			return e
+		}
+		require.Nil(t, e)
+		defer resp.Body.Close()
+		assert.Equal(t, 200, resp.StatusCode)
+		body, e := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, e)
+		assert.Equal(t, "pong", string(body))
+		passed = true
+		return nil
+	})
 
-	// send ping
+	assert.NoError(t, err)
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
+	assert.Equal(t, true, passed, "at least on ping passed")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
 
 	wg.Wait()
 }

backend/app/migrator/backup_test.go

@@ -16,7 +16,8 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {

backend/app/migrator/disqus.go

@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue

backend/app/migrator/native.go

@@ -2,23 +2,28 @@ package migrator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
+	"sync/atomic"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/syncs"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
+const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
 // {"version": 1, comments:[{...}\n,{}], meta: {meta}}
 // each comments starts from the new line
 type Native struct {
-	DataStore Store
+	DataStore  Store
+	Concurrent int
 }
 
 type meta struct {
@@ -45,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
 		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -70,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -91,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -99,8 +104,13 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, err
 	}
 
-	failed := 0
+	var failed, total, comments int64
-	total, comments := 0, 0
+
+	concurrent := defaultConcurrent
+	if n.Concurrent > 0 {
+		concurrent = n.Concurrent
+	}
+	grp := syncs.NewSizedGroup(concurrent, syncs.Preemptive)
 
 	for {
 		comment := store.Comment{}
@@ -112,27 +122,34 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		total++
 
 		if err != nil {
+			atomic.AddInt64(&failed, 1)
 			failed++
 			continue
 		}
 
-		if _, err = n.DataStore.Create(comment); err != nil {
+		// write comments in parallel
-			failed++
+		grp.Go(func(context.Context) {
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
+			if _, e := n.DataStore.Create(comment); e != nil {
-			continue
+				atomic.AddInt64(&failed, 1)
-		}
+				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
-		comments++
+				return
-		if comments%1000 == 0 {
+			}
-			log.Printf("[DEBUG] imported %d comments", comments)
+			num := atomic.AddInt64(&comments, 1)
-		}
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
+			}
+		})
+
 	}
 
+	grp.Wait()
+
 	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
+		return int(comments), errors.Errorf("failed to save %d comments", failed)
 	}
 	log.Printf("[INFO] imported %d comments from %d records", comments, total)
 
 	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
 
-	return comments, err
+	return int(comments), err
 }

backend/app/migrator/wordpress.go

@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue

backend/app/notify/notify.go

@@ -29,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment

backend/app/notify/telegram.go

@@ -42,7 +42,10 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	}
 	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
-	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	err := repeater.NewDefault(5, time.Millisecond*250).Do(ctx, func() error {
 		client := http.Client{Timeout: telegramTimeOut}
 		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
 		if err != nil {

backend/app/notify/telegram_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NotNil(t, err, "should failed")
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())

backend/app/rest/api/admin.go

@@ -55,10 +55,10 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	err := a.dataService.Delete(locator, id, store.SoftDelete)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment", rest.ErrInternal)
 		return
 	}
-	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
@@ -71,7 +71,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
 	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -88,7 +88,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 
 	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
 	if err != nil || len(ucomments) == 0 {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
 	}
 	render.Status(r, http.StatusOK)
@@ -103,7 +103,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token", rest.ErrActionRejected)
 		return
 	}
 
@@ -111,19 +111,19 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
 	if !claims.User.BoolAttr("delete_me") {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token", rest.ErrNoAccess)
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
+		avatarStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
 	}
@@ -147,7 +147,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -159,7 +159,7 @@ func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	users, err := a.dataService.Blocked(siteID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
 	}
 	render.JSON(w, r, users)
@@ -178,13 +178,14 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
 	if !roStatus {
 		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
+			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+				"read-only due the age", rest.ErrActionRejected)
 			return
 		}
 	}
 
 	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status", rest.ErrPostNotFound)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
@@ -198,7 +199,7 @@ func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
 
 	c, err := a.dataService.SetTitle(locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title", rest.ErrInternal)
 		return
 	}
 	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
@@ -215,7 +216,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	verifyStatus := r.URL.Query().Get("verified") == "1"
 
 	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
@@ -230,7 +231,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	pinStatus := r.URL.Query().Get("pin") == "1"
 
 	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))

backend/app/rest/api/admin_test.go

@@ -16,12 +16,13 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	R "github.com/go-pkgz/rest"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/go-pkgz/rest/cache"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestAdmin_Delete(t *testing.T) {
@@ -36,12 +37,33 @@ func TestAdmin_Delete(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
+	// check last comments
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments := []store.Comment{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+
+	// check multi count
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -52,6 +74,36 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
+
+	time.Sleep(250 * time.Millisecond)
+	// check last comments updated
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+
+	// check count updated
+	res, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah")
+	assert.Equal(t, 200, code)
+	b := map[string]interface{}{}
+	err = json.Unmarshal([]byte(res), &b)
+	assert.Nil(t, err)
+	t.Logf("%#v", b)
+	assert.Equal(t, 1.0, b["count"], "should report 1 comments")
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestAdmin_Title(t *testing.T) {
@@ -61,11 +113,13 @@ func TestAdmin_Title(t *testing.T) {
 	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -262,6 +316,7 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.True(t, comments.Comments[0].Deleted)
 
+	srv.Cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
@@ -392,6 +447,33 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
+func TestAdmin_ReadOnlyNoComments(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	client := http.Client{}
+
+	// set post to read-only
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	assert.NotNil(t, err)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&format=tree")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
+}
+
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()

backend/app/rest/api/migrator.go

@@ -56,13 +56,14 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	tmpfile, err := m.saveTemp(r.Body)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -78,25 +79,26 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	if err := r.ParseMultipartForm(20 * 1024 * 1024); err != nil { // 20M max memory, if bigger will make a file
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
 		return
 	}
 
 	file, _, err := r.FormFile("file")
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import from the request")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
 	tmpfile, err := m.saveTemp(file)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -155,7 +157,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
 		return
 	}
 }

backend/app/rest/api/migrator_test.go

@@ -3,10 +3,10 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
@@ -15,7 +15,7 @@ import (
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
+	"github.com/coreos/bbolt"
 	"github.com/go-chi/chi"
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
@@ -24,16 +24,14 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -50,19 +48,12 @@ func TestMigrator_Import(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -85,18 +76,12 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client := &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 }
+
 func TestMigrator_ImportFromWP(t *testing.T) {
-	srv, ds, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
@@ -113,34 +98,12 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.NoError(t, ds.Interface.Close())
-
-	tsAccess, _, teardownAccess := startupT(t)
-	defer teardownAccess()
-
-	res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
-	require.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-	t.Logf("%+v", comments)
-	assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
-		comments[0].Text)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -154,13 +117,12 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
@@ -179,17 +141,16 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
@@ -208,12 +169,13 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, teardown := prepImportSrv(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -226,13 +188,7 @@ func TestMigrator_Export(t *testing.T) {
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 
 	// check file mode
 	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
@@ -273,12 +229,27 @@ func TestMigrator_Export(t *testing.T) {
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
+func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
+}
+
+func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
+	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
 	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc = &Migrator{
+	svc := &Migrator{
 		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
 		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
 		NativeImporter:    &migrator.Native{DataStore: dataStore},
@@ -296,12 +267,12 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *http
 	am := a.Middleware()
 	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
 	ts = httptest.NewServer(routes)
-	return svc, dataStore, ts
+	teardown = func() {
-}
+		ts.Close()
-
+		require.NoError(t, b.Close())
-func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
+		_ = os.Remove(testDb)
-	ts.Close()
+	}
-	_ = os.Remove(testDb)
+	return ts, teardown
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -30,6 +30,7 @@ import (
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -44,6 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     *image.Service
 
 	WebRoot         string
 	RemarkURL       string
@@ -80,6 +82,7 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		err := s.httpServer.ListenAndServe()
@@ -89,7 +92,10 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		go func() {
@@ -106,7 +112,11 @@ func (s *Rest) Run(port int) {
 		m := s.makeAutocertManager()
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.lock.Unlock()
 
 		go func() {
@@ -183,13 +193,13 @@ func (s *Rest) routes() chi.Router {
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Flags(logger.All), logger.Log(log.Default()), logger.IPfn(ipFn), logger.Prefix("[INFO]"))
+		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
+		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
-		r.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
 
@@ -199,7 +209,8 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
-			rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
 		})
 
@@ -207,7 +218,8 @@ func (s *Rest) routes() chi.Router {
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
 			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+			ropen.Use(middleware.NoCache)
+			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
 				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
 			ropen.Get("/find", s.findCommentsCtrl)
 			ropen.Get("/id/{id}", s.commentByIDCtrl)
@@ -219,16 +231,25 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
-
-			ropen.Mount("/rss", s.rssRoutes())
 			ropen.Mount("/img", s.ImageProxy.Routes())
+			ropen.Mount("/rss", s.rssRoutes())
+		})
+
+		// open routes, cached
+		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(authMiddleware.Trace)
+			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
 			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+			rauth.Use(middleware.NoCache)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
 				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
 			rauth.Get("/user", s.userInfoCtrl)
 			rauth.Get("/userdata", s.userAllDataCtrl)
@@ -245,20 +266,34 @@ func (s *Rest) routes() chi.Router {
 			}
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
 			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+			rauth.Use(middleware.NoCache)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
 				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
 			rauth.Put("/comment/{id}", s.updateCommentCtrl)
 			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
+			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
 		})
+
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+		})
+
 	})
 
 	// respond to /robots.txt with the list of allowed paths
 	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
 		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+				"/img", "/avatar", "/picture"}
 			for i := range allowed {
 				allowed[i] = "Allow: /api/v1" + allowed[i]
 			}
@@ -281,20 +316,54 @@ func (s *Rest) routes() chi.Router {
 	return router
 }
 
+func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+
+	res = s.adminService.alterComments(comments, r) // apply admin's alteration
+
+	// prepare vote info for client view
+	vote := func(c store.Comment, r *http.Request) store.Comment {
+
+		c.Vote = 0 // default is "none" (not voted)
+
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			c.Votes = nil // hide voters list and don't set Vote for non-authed user
+			return c
+		}
+
+		if v, ok := c.Votes[user.ID]; ok {
+			if v {
+				c.Vote = 1
+			} else {
+				c.Vote = -1
+			}
+		}
+
+		c.Votes = nil // hide voters list
+		return c
+	}
+
+	for i, c := range res {
+		c = vote(c, r)
+		res[i] = c
+	}
+
+	return res
+}
+
 // serves static files from /web or embedded by statik
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 
 	var webFS http.Handler
 
 	statikFS, err := fs.New()
-	if err == nil {
-		log.Printf("[INFO] run file server for %s, embedded", root)
-		webFS = http.FileServer(statikFS)
-	}
 	if err != nil {
 		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
 		log.Printf("[INFO] run file server for %s, path %s", root, path)
 		webFS = http.FileServer(root)
+	} else {
+		log.Printf("[INFO] run file server for %s, embedded", root)
+		webFS = http.FileServer(statikFS)
 	}
 
 	origPath := path
@@ -326,7 +395,8 @@ func encodeJSONWithHTML(v interface{}) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
+func filterComments(comments []store.Comment, fn func(c store.Comment) bool) []store.Comment {
+	filtered := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			filtered = append(filtered, c)
@@ -339,9 +409,42 @@ func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (fi
 // admins will have different keys in order to prevent leak of admin-only data to regular users
 func URLKey(r *http.Request) string {
 	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
+	if user, err := rest.GetUserInfo(r); err == nil && user.Admin {
-		key = adminPrefix + key
+		key = adminPrefix + key // make separate cache key for admins
 	}
 	return key
 }
+
+// URLKeyWithUser gets url from request to use it as cache key and attaching user ID
+// admins will have different keys in order to prevent leak of admin-only data to regular users
+func URLKeyWithUser(r *http.Request) string {
+	adminPrefix := "admin!!"
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil {
+		if user.Admin {
+			key = adminPrefix + user.ID + "!!" + key // make separate cache key for admins
+		} else {
+			key = user.ID + "!!" + key // make separate cache key for authed users
+		}
+	}
+	return key
+}
+
+// rejectAnonUser is a middleware rejecting anonymous users
+func rejectAnonUser(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		if strings.HasPrefix(user.ID, "anonymous_") {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}

backend/app/rest/api/rest_private.go

@@ -28,7 +28,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -40,36 +40,36 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment.Orig = comment.Text // original comment text, prior to md render
 	if err := s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	comment = s.CommentFormatter.Format(comment)
 
 	// check if user blocked
 	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	if s.isReadOnly(comment.Locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	id, err := s.DataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment", rest.ErrInternal)
 		return
 	}
 
 	// DataService modifies comment
 	finalComment, err := s.DataService.Get(comment.Locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
@@ -95,7 +95,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -108,12 +108,13 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	var currComment store.Comment
 	var err error
 	if currComment, err = s.DataService.Get(locator, id); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
 
 	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+			"can not edit comments for other users", rest.ErrNoAccess)
 		return
 	}
 
@@ -126,15 +127,17 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	res, err := s.DataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
-		return
-	}
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID))
+	if err != nil {
+		code := s.parseError(err, rest.ErrCommentRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
+		return
+	}
+
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
@@ -158,19 +161,20 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	vote := r.URL.Query().Get("vote") == "1"
 
 	if s.isReadOnly(locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	// check if user blocked
 	if s.adminService.checkBlocked(locator.SiteID, user) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
+		code := s.parseError(err, rest.ErrVoteRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
 	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
@@ -183,7 +187,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info", rest.ErrInternal)
 		return
 	}
 
@@ -209,14 +213,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
+		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
+		if errUser != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
+		b, errUser := json.Marshal(comments)
-		if err != nil {
+		if errUser != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
@@ -228,7 +232,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	merr = multierror.Append(merr, write([]byte(`}`)))
 	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info", rest.ErrInternal)
 		return
 	}
 
@@ -258,7 +262,7 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 
 	tokenStr, err := s.Authenticator.TokenService().Token(claims)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
@@ -266,6 +270,31 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
+// POST /image - save image with form request
+func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"id": id})
+}
+
 func (s *Rest) isReadOnly(locator store.Locator) bool {
 	if s.ReadOnlyAge > 0 {
 		// check RO by age
@@ -275,3 +304,28 @@ func (s *Rest) isReadOnly(locator store.Locator) bool {
 	}
 	return s.DataService.IsReadOnly(locator) // ro manually
 }
+
+func (s *Rest) parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private_test.go

@@ -1,22 +1,37 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/rest"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
 func TestRest_Create(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -216,21 +231,34 @@ func TestRest_UpdateDelete(t *testing.T) {
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
+	// check multi count updated
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, b.StatusCode, string(body))
 
 	// comments returned by update
 	c2 := store.Comment{}
 	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, id, c2.ID)
 	assert.True(t, c2.Deleted)
 
@@ -243,6 +271,18 @@ func TestRest_UpdateDelete(t *testing.T) {
 	assert.Equal(t, "", c3.Text)
 	assert.Equal(t, "", c3.Orig)
 	assert.True(t, c3.Deleted)
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestRest_UpdateNotOwner(t *testing.T) {
@@ -264,7 +304,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	body, err := ioutil.ReadAll(b.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
+	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
 	client = http.Client{}
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
@@ -318,7 +358,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("admin", "password")
+		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -326,22 +366,65 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, 200, vote(1), "first vote allowed")
 	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"admin": true}, cr.Votes)
+	assert.Equal(t, 1, cr.Vote)
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
+	assert.Equal(t, 0, cr.Vote)
+
+	assert.Equal(t, 200, vote(-1), "opposite vote allowed one more time")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	assert.Equal(t, 400, vote(-1), "dbl vote not allowed")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	req, err := http.NewRequest("GET",
+		fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+	assert.NoError(t, err)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	cr = store.Comment{}
+	err = json.NewDecoder(resp.Body).Decode(&cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -407,13 +490,12 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
 
-	for i := 0; i < 478; i++ {
+	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
 		c.Timestamp = c.Timestamp.Add(time.Second)
 		_, err := srv.DataService.Create(c)
 		require.Nil(t, err)
 	}
-
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
@@ -429,7 +511,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
 		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
+	assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
@@ -464,3 +546,163 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	savePic := func(name string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 1462, len(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+}
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	svc.ImageService = &image.Service{
+		Store: &image.FileSystem{
+			Staging:  "/tmp/remark42/images.staging",
+			Location: "/tmp/remark42/images",
+			MaxSize:  2000,
+		},
+		TTL: time.Millisecond * 100,
+	}
+	svc.DataService.EditDuration = time.Millisecond * 100
+	svc.DataService.ImageService = svc.ImageService
+
+	uploadPicture := func(file string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.Contains(t, m["id"], ".png")
+		t.Logf(string(body))
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png")
+	id2 := uploadPicture("pic2.png")
+	id3 := uploadPicture("pic3.png")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	svc := Rest{}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := svc.parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}

backend/app/rest/api/rest_public.go

@@ -1,8 +1,9 @@
 package api
 
 import (
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -17,7 +18,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
 // find comments for given post. Returns in tree or plain formats, sorted
 func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
@@ -27,17 +28,20 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
 
-	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Find(locator, sort)
 		if e != nil {
-			return nil, e
+			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.adminService.alterComments(comments, r)
+		maskedComments := s.alterComments(comments, r)
 		var b []byte
 		switch r.URL.Query().Get("format") {
 		case "tree":
 			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			if tree.Nodes == nil { // eliminate json nil serialization
+				tree.Nodes = []*rest.Node{}
+			}
 			if s.DataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
@@ -53,7 +57,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -66,19 +70,19 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
 	user, err := rest.GetUserInfo(r)
 	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info", rest.ErrNoAccess)
 		return
 	}
 	comment.User = user
 	comment.Orig = comment.Text
 	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
@@ -101,7 +105,7 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info", rest.ErrPostNotFound)
 		return
 	}
 
@@ -126,14 +130,14 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments", rest.ErrInternal)
 		return
 	}
 
@@ -153,10 +157,10 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
+	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -182,13 +186,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
-	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID)
+	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.User(siteID, userID, limit, 0)
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		count, e := s.DataService.UserCount(siteID, userID)
 		if e != nil {
@@ -199,7 +203,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -221,7 +225,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		Auth           []string `json:"auth_providers"`
 		LowScore       int      `json:"low_score"`
 		CriticalScore  int      `json:"critical_score"`
+		PositiveScore  bool     `json:"positive_score"`
 		ReadOnlyAge    int      `json:"readonly_age"`
+		MaxImageSize   int      `json:"max_image_size"`
 	}
 
 	cnf := config{
@@ -232,7 +238,9 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		AdminEmail:     s.DataService.AdminStore.Email(siteID),
 		LowScore:       s.ScoreThresholds.Low,
 		CriticalScore:  s.ScoreThresholds.Critical,
+		PositiveScore:  s.DataService.PositiveScore,
 		ReadOnlyAge:    s.ReadOnlyAge,
+		MaxImageSize:   s.ImageService.Store.SizeLimit(),
 	}
 
 	cnf.Auth = []string{}
@@ -252,29 +260,26 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	count, err := s.DataService.Count(locator)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
 	}
 	render.JSON(w, r, R.JSON{"count": count, "locator": locator})
 }
 
-// POST /count?site=siteID - get number of comments for posts from post body
+// POST /counts?site=siteID - get number of comments for posts from post body
 func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
 		return
 	}
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
+	h := sha1.Sum([]byte(k)) //nolint
-	if _, err := hasher.Write([]byte(k)); err != nil {
+	sha := base64.URLEncoding.EncodeToString(h[:])
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
+
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
@@ -285,7 +290,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -317,7 +322,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -325,3 +330,46 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't render posts lits for site %s", siteID)
 	}
 }
+
+// GET /picture/{user}/{id} - get picture
+func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
+	imgRdr, size, err := s.ImageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer imgRdr.Close()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}

backend/app/rest/api/rest_public_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -69,8 +70,12 @@ func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -83,10 +88,10 @@ func TestRest_Find(t *testing.T) {
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
+	comments = commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
+	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
@@ -194,6 +199,10 @@ func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "[]\n", res, "empty last should return empty list")
+
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
@@ -204,7 +213,7 @@ func TestRest_Last(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	id2 := addComment(t, c2, ts)
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
@@ -227,6 +236,7 @@ func TestRest_Last(t *testing.T) {
 
 	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
 	assert.Nil(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
@@ -367,6 +377,37 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 3, pi[1].Count)
 }
 
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
+}
+
 func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -382,7 +423,9 @@ func TestRest_Config(t *testing.T) {
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
 	assert.Equal(t, -10., j["critical_score"])
+	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
+	assert.Equal(t, 10000., j["max_image_size"])
 	t.Logf("%+v", j)
 }
 
@@ -431,5 +474,5 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }

backend/app/rest/api/rest_test.go

@@ -4,10 +4,13 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -22,19 +25,22 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
 var testHTML = "/tmp/test-remark.html"
 var getStartedHTML = "/tmp/getstarted.html"
 
 var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -175,10 +181,88 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 	srv.Shutdown()
 }
 
+func TestRest_rejectAnonUser(t *testing.T) {
+
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello")
+	}))))
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
+
+	resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
+
+	resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
+}
+
+func Test_URLKey(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKey(r))
+		})
+	}
+
+}
+
+func Test_URLKeyWithUser(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "user!!http://example.com/1"},
+		{"http://example.com/2", store.User{ID: "user2"}, "user2!!http://example.com/2"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!user!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKeyWithUser(r))
+		})
+	}
+
+}
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+
+	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
+	os.Remove(testDb)
+	os.Remove(testHTML)
+	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
+
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 
+	memCache, err := cache.NewMemoryCache()
+	assert.NoError(t, err)
+
 	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
@@ -198,10 +282,18 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
 		}),
-		Cache:     &cache.Nop{},
+		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+				Staging:    "/tmp/pics-remark42/staging",
+			},
+			TTL: time.Millisecond * 100,
+		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -223,15 +315,30 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 
 	teardown = func() {
 		ts.Close()
-		srv.DataService.Close()
+		require.NoError(t, srv.DataService.Close())
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
+// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+func fakeAuth(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("fake_id") != "" {
+			r = rest.SetUserInfo(r, store.User{
+				ID:   r.URL.Query().Get("fake_id"),
+				Name: r.URL.Query().Get("fake_name"),
+			})
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
 	require.Nil(t, err)
@@ -241,6 +348,14 @@ func get(t *testing.T, url string) (string, int) {
 	return string(body), r.StatusCode
 }
 
+func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+	if token != "" {
+		r.Header.Set("X-JWT", token)
+	}
+	return client.Do(r)
+}
+
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)

backend/app/rest/api/rss.go

@@ -16,8 +16,8 @@ import (
 )
 
 const maxRssItems = 20
-const maxLastCommentsReply = 1000
+const maxLastCommentsReply = 5000
-const maxReplyDuration = 24 * time.Hour
+const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
@@ -41,8 +41,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments)
+		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
@@ -50,14 +50,14 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrPostNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -73,9 +73,9 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
+		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
@@ -83,13 +83,13 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -100,18 +100,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
+	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
 	data, err := s.Cache.Get(key, func() (res []byte, e error) {
 		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.adminService.alterComments(comments, r)
+		comments = s.alterComments(comments, r)
 		replies := []store.Comment{}
 		for _, c := range comments {
 			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
 				break
 			}
+			if c.User.ID != userID {
+				userName = c.User.Name
+			}
 			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
 				var pc store.Comment
 				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
@@ -123,7 +127,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		rss, e := s.toRssFeed(siteID, replies)
+		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
@@ -131,19 +135,22 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
+func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
+	if description == "" {
+		description = "comment updates"
+	}
 	lastCommentTS := time.Unix(0, 0)
 	if len(comments) > 0 {
 		lastCommentTS = comments[0].Timestamp
@@ -152,7 +159,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 	feed := &feeds.Feed{
 		Title:       "Remark42 comments",
 		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
+		Description: description,
 		Created:     lastCommentTS,
 	}
 

backend/app/rest/api/rss_test.go

@@ -38,7 +38,7 @@ func TestServer_RssPost(t *testing.T) {
         <channel>
             <title>Remark42 comments</title>
             <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
+            <description>post comments for https://radio-t.com/blah1</description>
             <pubDate>%s</pubDate>
             <item>
 		      <title>developer one</title>
@@ -93,7 +93,7 @@ func TestServer_RssSite(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>radio-t</link>
-		    <description>comment updates</description>
+		    <description>site comment for radio-t</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -156,7 +156,7 @@ func TestServer_RssWithReply(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
+		    <description>post comments for https://radio-t.com/blah10</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one &gt; developer one</title>
@@ -242,7 +242,7 @@ func TestServer_RssReplies(t *testing.T) {
 	    <channel>
 	        <title>Remark42 comments</title>
 	        <link>radio-t</link>
-	        <description>comment updates</description>
+	        <description>replies to user2</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>

backend/app/rest/httperrors.go

@@ -9,16 +9,40 @@ import (
 
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest"
+)
+
+// All error codes for UI mapping and translation
+const (
+	ErrInternal           = 0  // any internal error
+	ErrCommentNotFound    = 1  // can't find comment
+	ErrDecode             = 2  // failed to unmarshal incoming request
+	ErrNoAccess           = 3  // rejected by auth
+	ErrCommentValidation  = 4  // validation failed
+	ErrPostNotFound       = 5  // can't find post
+	ErrSiteNotFound       = 6  // can't find site
+	ErrUserBlocked        = 7  // user blocked
+	ErrReadOnly           = 8  // write failed on read only
+	ErrCommentRejected    = 9  // general error on rejected comment change
+	ErrCommentEditExpired = 10 // too late for edit
+	ErrCommentEditChanged = 11 // parent comment cannot be changed
+	ErrVoteRejected       = 12 // general error on vote rejected
+	ErrVoteSelf           = 13 // vote for own comment
+	ErrVoteDbl            = 14 // already voted for the comment
+	ErrVoteMax            = 15 // too many votes for the comment
+	ErrVoteMinScore       = 16 // min score reached for the comment
+	ErrActionRejected     = 17 // general error for rejected actions
+	ErrAssetNotFound      = 18 // requested file not found
 )
 
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
+func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
+	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
-	render.Status(r, code)
+	render.Status(r, httpStatusCode)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
+	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
 
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
+func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details string, errCode int) string {
 	uinfoStr := ""
 	if user, e := GetUserInfo(r); e == nil {
 		uinfoStr = user.Name + "/" + user.ID + " - "
@@ -40,5 +64,6 @@ func errDetailsMsg(r *http.Request, code int, err error, details string) string
 	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
 		remoteIP = remoteIP[:pos]
 	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
+		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -17,7 +17,7 @@ func TestSendErrorJSON(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
+			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456", 123)
 			return
 		}
 		w.WriteHeader(404)
@@ -33,7 +33,7 @@ func TestSendErrorJSON(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
+	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
 func TestErrorDetailsMsg(t *testing.T) {
@@ -41,8 +41,8 @@ func TestErrorDetailsMsg(t *testing.T) {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
 		require.Nil(t, err)
 		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
 	}
 	callerFn()
 }
@@ -53,8 +53,8 @@ func TestErrorDetailsMsgWithUser(t *testing.T) {
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
 		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"context"
 	"encoding/base64"
 	"io"
 	"net/http"
@@ -22,6 +23,7 @@ type Image struct {
 	RemarkURL string
 	RoutePath string
 	Enabled   bool
+	Timeout   time.Duration
 }
 
 // Convert all img src links without https to proxied links
@@ -47,19 +49,31 @@ func (p Image) Routes() chi.Router {
 	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
 		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
 			return
 		}
 
+		timeout := 60 * time.Second // default
+		if p.Timeout > 0 {
+			timeout = p.Timeout
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+
 		client := http.Client{Timeout: 30 * time.Second}
 		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(func() error {
+		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
 			var e error
-			resp, e = client.Get(string(src))
+			req, e := http.NewRequest("GET", string(src), nil)
+			if e != nil {
+				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
+			}
+			resp, e = client.Do(req.WithContext(ctx))
 			return e
 		})
 		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
 			return
 		}
 		defer func() {

backend/app/rest/proxy/image_test.go

@@ -3,9 +3,12 @@ package proxy
 import (
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -87,6 +90,25 @@ func TestImage_Routes(t *testing.T) {
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
+func TestImage_RoutesTimedOut(t *testing.T) {
+	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+	router := img.Routes()
+
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	t.Log(string(b))
+	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
+}
+
 func TestPicture_Convert(t *testing.T) {
 	img := Image{Enabled: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
@@ -110,7 +132,13 @@ func imgHTTPServer(t *testing.T) *httptest.Server {
 			t.Log("http img request", r.URL)
 			w.Header().Add("Content-Length", "123")
 			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			assert.NoError(t, err)
+			return
+		}
+		if r.URL.Path == "/image/img-slow.png" {
+			time.Sleep(500 * time.Millisecond)
+			w.WriteHeader(500)
 			return
 		}
 		t.Log("http img request - not found", r.URL)

backend/app/rest/tree.go

@@ -111,8 +111,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 }
 
 // filter returns comments for parentID
-func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) (f []store.Comment) {
+func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) []store.Comment {
-
+	f := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			f = append(f, c)
@@ -151,6 +151,18 @@ func (t *Tree) sortNodes(sortType string) {
 			}
 			return t.Nodes[i].Comment.Score < t.Nodes[j].Comment.Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortType, "-") {
+				if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+					return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+				}
+				return t.Nodes[i].Comment.Controversy > t.Nodes[j].Comment.Controversy
+			}
+			if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+				return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+			}
+			return t.Nodes[i].Comment.Controversy < t.Nodes[j].Comment.Controversy
+
 		default:
 			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
 		}

backend/app/rest/tree_test.go

@@ -96,15 +96,15 @@ func TestTreeSortNodes(t *testing.T) {
 	comments := []store.Comment{
 		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
+		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2, Controversy: 10},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
+		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3, Controversy: 5},
 		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
 		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
 		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
 		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
 		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
+		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2, Controversy: 7},
 		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
 		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
@@ -140,6 +140,19 @@ func TestTreeSortNodes(t *testing.T) {
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
+	res = MakeTree(comments, "+controversy", 0)
+	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
+
+	res = MakeTree(comments, "-controversy", 0)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
+
 	res = MakeTree(comments, "undefined", 0)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)

backend/app/store/comment.go

@@ -10,19 +10,21 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID        string          `json:"id" bson:"_id"`
+	ID          string          `json:"id" bson:"_id"`
-	ParentID  string          `json:"pid"`
+	ParentID    string          `json:"pid"`
-	Text      string          `json:"text"`
+	Text        string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
+	Orig        string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
+	User        User            `json:"user"`
-	Locator   Locator         `json:"locator"`
+	Locator     Locator         `json:"locator"`
-	Score     int             `json:"score"`
+	Score       int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
+	Votes       map[string]bool `json:"votes,omitempty"`
-	Timestamp time.Time       `json:"time" bson:"time"`
+	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
-	Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Controversy float64         `json:"controversy,omitempty"`
-	Pin       bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Timestamp   time.Time       `json:"time" bson:"time"`
-	Deleted   bool            `json:"delete,omitempty" bson:"delete"`
+	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	PostTitle string          `json:"title,omitempty" bson:"title"`
+	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string          `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post

backend/app/store/engine/bolt_accessor.go

@@ -74,6 +74,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
 	}
 	return &result, nil
 }
@@ -155,7 +156,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 
 		return bucket.ForEach(func(k, v []byte) error {
 			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
+			if e = json.Unmarshal(v, &comment); e != nil {
 				return errors.Wrap(e, "failed to unmarshal")
 			}
 			comments = append(comments, comment)
@@ -163,13 +164,15 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 		})
 	})
 
-	comments = sortComments(comments, sortFld)
+	comments = SortComments(comments, sortFld)
 	return comments, err
 }
 
 // Last returns up to max last comments for given siteID
 func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
 
+	comments = []store.Comment{}
+
 	if max > lastLimit || max == 0 {
 		max = lastLimit
 	}
@@ -193,7 +196,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
 			}
 
 			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
+			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
 				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
 				continue
 			}
@@ -333,11 +336,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.
 
 	// retrieve comments for refs
 	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
+		url, commentID, errParse := b.parseRef([]byte(v))
-		if e != nil {
+		if errParse != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
 		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
+		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
 			comments = append(comments, c)
 		}
 	}

backend/app/store/engine/bolt_accessor_test.go

@@ -16,8 +16,8 @@ import (
 var testDb = "test-remark.db"
 
 func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	var b = prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -37,8 +37,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
 }
 
 func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	var b = prep(t)
+	defer teardown()
 
 	comment := store.Comment{
 		ID:        "id-ro",
@@ -61,8 +61,8 @@ func TestBoltDB_CreateReadOnly(t *testing.T) {
 }
 
 func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
 
 	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
 	assert.Nil(t, err)
@@ -80,8 +80,9 @@ func TestBoltDB_Get(t *testing.T) {
 }
 
 func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
+
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
 	assert.Nil(t, err)
@@ -107,8 +108,8 @@ func TestBoltDB_Put(t *testing.T) {
 }
 
 func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
 
 	res, err := b.Last("radio-t", 0)
 	assert.Nil(t, err)
@@ -125,8 +126,8 @@ func TestBoltDB_Last(t *testing.T) {
 }
 
 func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
 
 	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -141,8 +142,8 @@ func TestBoltDB_Count(t *testing.T) {
 }
 
 func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // two comments for https://radio-t.com
-	b := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	// add one more for https://radio-t.com/2
 	comment := store.Comment{
@@ -181,8 +182,8 @@ func TestBoltDB_List(t *testing.T) {
 }
 
 func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
+	b, teardown := prep(t) // two comments for https://radio-t.com
-	b := prep(t) // two comments for https://radio-t.com
+	defer teardown()
 
 	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
 
@@ -224,8 +225,8 @@ func TestBoltDB_Info(t *testing.T) {
 }
 
 func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
 
 	res, err := b.User("radio-t", "user1", 5, 0)
 	assert.Nil(t, err)
@@ -253,7 +254,11 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 	os.Remove(testDb)
 	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
-	defer os.Remove(testDb)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}()
 
 	c := store.Comment{
 		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -303,8 +308,9 @@ func TestBoltDB_GetForUserPagination(t *testing.T) {
 }
 
 func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
+	var b, teardown = prep(t)
-	b := prep(t)
+	defer teardown()
+
 	count, err := b.UserCount("radio-t", "user1")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, count)
@@ -343,12 +349,12 @@ func TestBoltDB_New(t *testing.T) {
 }
 
 // makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
 	os.Remove(testDb)
 
 	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
 	assert.Nil(t, err)
-	b := boltStore
+	b = boltStore
 
 	comment := store.Comment{
 		ID:        "id-1",
@@ -370,5 +376,9 @@ func prep(t *testing.T) *BoltDB {
 	_, err = b.Create(comment)
 	assert.Nil(t, err)
 
-	return b
+	teardown = func() {
+		require.NoError(t, b.Close())
+		os.Remove(testDb)
+	}
+	return b, teardown
 }

backend/app/store/engine/bolt_admin.go

@@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
 		}
 
 		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
+		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
 			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
+		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
 			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
 		}
 
 		// delete from "last" bucket
 		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
+		if err = lastBkt.Delete([]byte(commentID)); err != nil {
 			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
@@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
 			return nil
 		}
 
-		until, err := time.Parse(tsNano, string(val))
+		until, e := time.Parse(tsNano, string(val))
-		if err != nil {
+		if e != nil {
 			blocked = false
 			return nil
 		}
@@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(blocksBucketName))
 		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
+			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
+			if errParse != nil {
-				return errors.Wrap(e, "can't parse block ts")
+				return errors.Wrap(errParse, "can't parse block ts")
 			}
 			if time.Now().Before(ts) {
 				// get user name from comment user section
 				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
+				userComments, errUser := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
+				if errUser == nil && len(userComments) > 0 {
 					userName = userComments[0].User.Name
 				}
 				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})

backend/app/store/engine/bolt_admin_test.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	"os"
 	"testing"
 	"time"
 
@@ -12,14 +11,19 @@ import (
 )
 
 func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(res), "initially 2 comments")
 
+	count, err := b.Count(loc)
+	require.NoError(t, err)
+	assert.Equal(t, 2, count, "count=2 initially")
+
 	err = b.Delete(loc, res[0].ID, store.SoftDelete)
 	assert.Nil(t, err)
 
@@ -37,6 +41,10 @@ func TestBoltAdmin_Delete(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
 
+	count, err = b.Count(loc)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
 	err = b.Delete(loc, "123456", store.SoftDelete)
 	assert.NotNil(t, err)
 
@@ -50,8 +58,9 @@ func TestBoltAdmin_Delete(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -70,8 +79,9 @@ func TestBoltAdmin_DeleteHard(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
 	res, err := b.Find(loc, "time")
@@ -94,8 +104,10 @@ func TestBoltAdmin_DeleteAll(t *testing.T) {
 }
 
 func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
+
 	err := b.DeleteUser("radio-t", "user1")
 	require.NoError(t, err)
 
@@ -122,8 +134,9 @@ func TestBoltAdmin_DeleteUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 
@@ -142,8 +155,10 @@ func TestBoltAdmin_BlockUser(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
+
 	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
 	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
@@ -152,8 +167,9 @@ func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
 }
 
 func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
 	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
@@ -178,8 +194,9 @@ func TestBoltAdmin_BlockList(t *testing.T) {
 }
 
 func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
 
@@ -198,8 +215,9 @@ func TestBoltAdmin_ReadOnly(t *testing.T) {
 }
 
 func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
+
-	b := prep(t)
+	b, teardown := prep(t)
+	defer teardown()
 
 	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
 

backend/app/store/engine/engine.go

@@ -62,8 +62,8 @@ const (
 	userLimit = 500
 )
 
-// sortComments is for engines can't sort data internally
+// SortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
+func SortComments(comments []store.Comment, sortFld string) []store.Comment {
 	sort.Slice(comments, func(i, j int) bool {
 		switch sortFld {
 		case "+time", "-time", "time", "+active", "-active", "active":
@@ -84,6 +84,18 @@ func sortComments(comments []store.Comment, sortFld string) []store.Comment {
 			}
 			return comments[i].Score < comments[j].Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortFld, "-") {
+				if comments[i].Controversy == comments[j].Controversy {
+					return comments[i].Timestamp.Before(comments[j].Timestamp)
+				}
+				return comments[i].Controversy > comments[j].Controversy
+			}
+			if comments[i].Controversy == comments[j].Controversy {
+				return comments[i].Timestamp.Before(comments[j].Timestamp)
+			}
+			return comments[i].Controversy < comments[j].Controversy
+
 		default:
 			return comments[i].Timestamp.Before(comments[j].Timestamp)
 		}

backend/app/store/engine/engine_test.go

@@ -11,33 +11,45 @@ import (
 
 func TestEngine_sortComments(t *testing.T) {
 	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
+		{ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
+		{ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
+		{ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
+		{ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
 	}
 
-	sortComments(cc, "+time")
+	SortComments(cc, "+time")
 	assert.Equal(t, "1", cc[0].ID)
 	assert.Equal(t, "2", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-time")
+	SortComments(cc, "-time")
 	assert.Equal(t, "4", cc[0].ID)
 	assert.Equal(t, "3", cc[1].ID)
 	assert.Equal(t, "2", cc[2].ID)
 	assert.Equal(t, "1", cc[3].ID)
 
-	sortComments(cc, "score")
+	SortComments(cc, "score")
 	assert.Equal(t, "2", cc[0].ID)
 	assert.Equal(t, "1", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-score")
+	SortComments(cc, "-score")
 	assert.Equal(t, "3", cc[0].ID)
 	assert.Equal(t, "4", cc[1].ID)
 	assert.Equal(t, "1", cc[2].ID)
 	assert.Equal(t, "2", cc[3].ID)
+
+	SortComments(cc, "controversy")
+	assert.Equal(t, "1", cc[0].ID)
+	assert.Equal(t, "4", cc[1].ID)
+	assert.Equal(t, "2", cc[2].ID)
+	assert.Equal(t, "3", cc[3].ID)
+
+	SortComments(cc, "-controversy")
+	assert.Equal(t, "3", cc[0].ID)
+	assert.Equal(t, "2", cc[1].ID)
+	assert.Equal(t, "1", cc[2].ID)
+	assert.Equal(t, "4", cc[3].ID)
 }

backend/app/store/engine/mongo.go

@@ -230,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	for _, m := range metas {
+	for _, meta := range metas {
-		ids = append(ids, m.ID)
+		ids = append(ids, meta.ID)
 	}
 	return ids, nil
 }

backend/app/store/image/fs_store.go

@@ -0,0 +1,175 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"io/ioutil"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+	MaxHeight  int
+	MaxWidth   int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	data, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
+	}
+	if len(data) > f.MaxSize {
+		return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
+	}
+
+	// read header first, needs it to check if data is valid png/gif/jpeg
+	if !isValidImage(data[:512]) {
+		return "", errors.Errorf("file %s is not in allowed format", fileName)
+	}
+
+	data, resized := resize(data, f.MaxWidth, f.MaxHeight)
+
+	id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+	if resized { // resized also converted to png
+		id = strings.TrimSuffix(id, filepath.Ext(id)) + ".png"
+		dst = f.location(f.Staging, id)
+	}
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, fileName, len(data))
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", path, age)
+			return os.Remove(path)
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// SizeLimit returns max size of allowed image
+func (f *FileSystem) SizeLimit() int {
+	return f.MaxSize
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}

backend/app/store/image/fs_store_test.go

@@ -0,0 +1,308 @@
+package image
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveWithResize(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 32, 32
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1142, len(data))
+}
+
+func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 400, 300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 10786, len(data))
+}
+
+func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 1400, 1300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".jpg")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 23983, len(data))
+}
+
+func TestFsStore_WrongFormat(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
+	assert.EqualError(t, err, "file file1.png is not in allowed format")
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 2000
+	_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string) (path string) {
+		id, err := svc.Save(file, user, gopherPNG())
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		assert.Equal(t, 1462, len(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = &FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    1500,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/image.go

@@ -0,0 +1,201 @@
+// Package image handles storing, resizing and retrieval of images
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
+package image
+
+//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
+
+import (
+	"bytes"
+	"context"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	"image/png"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/rs/xid"
+	"golang.org/x/image/draw"
+)
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+	SizeLimit() int                                                          // max image size
+}
+
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	idsFn func() (ids []string)
+	TS    time.Time
+}
+
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil || s == nil {
+		return
+	}
+
+	s.once.Do(func() {
+		log.Printf("[DEBUG] image submitter activated")
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
+				}
+			}
+			log.Printf("[INFO] image submitter terminated")
+		}()
+	})
+
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
+}
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
+		}
+	}
+}
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
+// biggest side (width or height) preserving aspect ratio.
+// Returns original data if resizing is not needed or failed.
+// If resized the result will be for png format and ok flag will be true.
+func resize(data []byte, limitW, limitH int) ([]byte, bool) {
+	if data == nil || limitW <= 0 || limitH <= 0 {
+		return data, false
+	}
+
+	src, _, err := image.Decode(bytes.NewBuffer(data))
+	if err != nil {
+		log.Printf("[WARN] can't decode image, %s", err)
+		return data, false
+	}
+
+	bounds := src.Bounds()
+	w, h := bounds.Dx(), bounds.Dy()
+	if w <= limitW && h <= limitH || w <= 0 || h <= 0 {
+		log.Printf("[DEBUG] resizing image is smaller that the limit or has 0 size")
+		return data, false
+	}
+
+	newW, newH := getProportionalSizes(w, h, limitW, limitH)
+	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
+	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+
+	var out bytes.Buffer
+	if err = png.Encode(&out, m); err != nil {
+		log.Printf("[WARN] can't encode resized image to png, %s", err)
+		return data, false
+	}
+	return out.Bytes(), true
+}
+
+// getProportionalSizes returns width and height resized by both dimensions proportionally
+func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
+
+	if srcW <= limitW && srcH <= limitH {
+		return srcW, srcH
+	}
+
+	ratioW := float64(srcW) / float64(limitW)
+	propH := float64(srcH) / ratioW
+
+	ratioH := float64(srcH) / float64(limitH)
+	propW := float64(srcW) / ratioH
+
+	if int(propH) > limitH {
+		return int(propW), limitH
+	}
+
+	return limitW, int(propH)
+}
+
+// check if file f is a valid image format, i.e. gif, png, jpeg or webp
+func isValidImage(b []byte) bool {
+	ct := http.DetectContentType(b)
+	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
+}
+
+// guid makes a globally unique id
+func guid() string {
+	return xid.New().String()
+}

backend/app/store/image/image_mock.go

@@ -0,0 +1,105 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package image
+
+import context "context"
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import time "time"
+
+// MockStore is an autogenerated mock type for the Store type
+type MockStore struct {
+	mock.Mock
+}
+
+// Cleanup provides a mock function with given fields: ctx, ttl
+func (_m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
+	ret := _m.Called(ctx, ttl)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, time.Duration) error); ok {
+		r0 = rf(ctx, ttl)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Commit provides a mock function with given fields: id
+func (_m *MockStore) Commit(id string) error {
+	ret := _m.Called(id)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(id)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Load provides a mock function with given fields: id
+func (_m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	ret := _m.Called(id)
+
+	var r0 io.ReadCloser
+	if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(io.ReadCloser)
+		}
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(string) int64); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string) error); ok {
+		r2 = rf(id)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: fileName, userID, r
+func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) {
+	ret := _m.Called(fileName, userID, r)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok {
+		r0 = rf(fileName, userID, r)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok {
+		r1 = rf(fileName, userID, r)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SizeLimit provides a mock function with given fields:
+func (_m *MockStore) SizeLimit() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}

backend/app/store/image/image_test.go

@@ -0,0 +1,146 @@
+package image
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"io/ioutil"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{ImageAPI: "/blah/"}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
+func TestService_Cleanup(t *testing.T) {
+	store := MockStore{}
+	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
+
+	svc := Service{Store: &store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+	store.AssertNumberOfCalls(t, "Cleanup", 10)
+}
+
+func TestService_Submit(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 0)
+	time.Sleep(time.Millisecond * 150)
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_Close(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 3)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_resize(t *testing.T) {
+
+	// Reader is nil.
+	resized, ok := resize(nil, 100, 100)
+	assert.Nil(t, resized)
+	assert.False(t, ok)
+
+	// Negative limit error.
+	resized, ok = resize([]byte("some picture bin data"), -1, -1)
+	require.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("some picture bin data"))
+	assert.False(t, ok)
+
+	// Decode error.
+	resized, ok = resize([]byte("invalid image content"), 100, 100)
+	assert.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("invalid image content"))
+	assert.False(t, ok)
+
+	cases := []struct {
+		file   string
+		wr, hr int
+	}{
+		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
+		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
+	}
+
+	for _, c := range cases {
+		img, err := ioutil.ReadFile(c.file)
+		require.Nil(t, err, "can't open test file %s", c.file)
+
+		// No need for resize, image dimensions are smaller than resize limit.
+		resized, ok = resize(img, 800, 800)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.Equal(t, resized, img)
+		assert.False(t, ok)
+
+		// Resizing to half of width. Check resized image format PNG.
+		resized, ok = resize(img, 400, 400)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.True(t, ok)
+
+		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
+		assert.Nil(t, err, "file %s", c.file)
+		assert.Equal(t, "png", format, "file %s", c.file)
+		bounds := imgRz.Bounds()
+		assert.Equal(t, c.wr, bounds.Dx(), "file %s", c.file)
+		assert.Equal(t, c.hr, bounds.Dy(), "file %s", c.file)
+	}
+
+}
+
+func TestGetProportionalSizes(t *testing.T) {
+	tbl := []struct {
+		inpW, inpH     int
+		limitW, limitH int
+		resW, resH     int
+	}{
+		{10, 20, 50, 25, 10, 20},
+		{400, 200, 50, 25, 50, 25},
+		{100, 100, 50, 25, 25, 25},
+		{100, 200, 50, 25, 12, 25},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			resW, resH := getProportionalSizes(tt.inpW, tt.inpH, tt.limitW, tt.limitH)
+			assert.Equal(t, tt.resW, resW, "width")
+			assert.Equal(t, tt.resH, resH, "height")
+		})
+	}
+}

backend/app/store/service/restricted_words_test.go

@@ -1,8 +1,9 @@
 package service
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMatcher_Tokenize(t *testing.T) {

backend/app/store/service/service.go

@@ -1,15 +1,18 @@
 package service
 
 import (
+	"math"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
-	"github.com/hashicorp/go-multierror"
+	multierror "github.com/hashicorp/go-multierror"
-	"github.com/patrickmn/go-cache"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -23,8 +26,10 @@ type DataStore struct {
 	AdminStore             admin.Store
 	MaxCommentSize         int
 	MaxVotes               int
+	PositiveScore          bool
 	TitleExtractor         *TitleExtractor
 	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -75,18 +80,44 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", ErrRestrictedWordsFound
 	}
 
-	// keep input title and set to extracted if missing
+	func() { // keep input title and set to extracted if missing
-	if s.TitleExtractor != nil && comment.PostTitle == "" {
+		if s.TitleExtractor == nil || comment.PostTitle != "" {
-		if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
+			return
-			comment.PostTitle = title
-		} else {
-			log.Printf("[WARN] failed to set title, %v", err)
 		}
-	}
+		title, e := s.TitleExtractor.Get(comment.Locator.URL)
+		if e != nil {
+			log.Printf("[WARN] failed to set title, %v", e)
+			return
+		}
+		comment.PostTitle = title
+	}()
 
+	s.submitImages(comment)
 	return s.Interface.Create(comment)
 }
 
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			return nil
+		}
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+}
+
 // prepareNewComment sets new comment fields, hashing and sanitizing data
 func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, error) {
 	// fill ID and time if empty
@@ -123,8 +154,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool)
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
 	comment, err = s.Get(locator, commentID)
@@ -154,6 +185,10 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		return comment, errors.Errorf("maximum number of votes exceeded for comment %s", commentID)
 	}
 
+	if s.PositiveScore && comment.Score <= 0 && !val {
+		return comment, errors.Errorf("minimal score reached for comment %s", commentID)
+	}
+
 	// reset vote if user changed to opposite
 	if voted && v != val {
 		delete(comment.Votes, userID)
@@ -171,9 +206,36 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		comment.Score--
 	}
 
+	comment.Vote = 0
+	if v, ok := comment.Votes[userID]; ok {
+		if v {
+			comment.Vote = 1
+		} else {
+			comment.Vote = -1
+		}
+	}
+
+	comment.Controversy = s.controversy(s.upsAndDowns(comment))
+
 	return comment, s.Put(locator, comment)
 }
 
+// controversy calculates controversial index of votes
+// source - https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L60
+func (s *DataStore) controversy(ups, downs int) float64 {
+
+	if downs <= 0 || ups <= 0 {
+		return 0
+	}
+
+	magnitude := ups + downs
+	balance := float64(downs) / float64(ups)
+	if ups <= downs {
+		balance = float64(ups) / float64(downs)
+	}
+	return math.Pow(float64(magnitude), balance)
+}
+
 // EditRequest contains fields needed for comment update
 type EditRequest struct {
 	Text    string
@@ -390,8 +452,45 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
 	return errs.ErrorOrNil()
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
+// Find wraps engine's Find call and alter results if needed
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+func (s *DataStore) Find(locator store.Locator, sort string) ([]store.Comment, error) {
+	comments, err := s.Interface.Find(locator, sort)
+	if err != nil {
+		return comments, err
+	}
+
+	changedSort := false
+	// set votes controversy for comments added prior to #274
+	for i, c := range comments {
+		if c.Controversy == 0 && len(c.Votes) > 0 {
+			comments[i].Controversy = s.controversy(s.upsAndDowns(c))
+			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+				changedSort = true
+			}
+		}
+	}
+
+	// resort commits if altered
+	if changedSort {
+		comments = engine.SortComments(comments, sort)
+	}
+
+	return comments, nil
+}
+
+func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
+	for _, v := range c.Votes {
+		if v {
+			ups++
+			continue
+		}
+		downs++
+	}
+	return ups, downs
+}
+
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()

backend/app/store/service/service_test.go

@@ -13,19 +13,22 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 )
 
 var testDb = "/tmp/test-remark.db"
 
 func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -45,11 +48,11 @@ func TestService_CreateFromEmpty(t *testing.T) {
 	assert.Equal(t, "user", res.User.ID)
 	assert.Equal(t, "name", res.User.Name)
 	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, map[string]bool{}, res.Votes)
+	assert.Equal(t, map[string]bool(nil), res.Votes)
 }
 
 func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -76,7 +79,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
@@ -106,7 +109,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 }
 
 func TestService_SetTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -114,11 +117,13 @@ func TestService_SetTitle(t *testing.T) {
 			w.WriteHeader(404)
 		}
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -145,6 +150,8 @@ func TestService_SetTitle(t *testing.T) {
 	t.Logf("%+v", res)
 	assert.Equal(t, "", res.PostTitle)
 
+	b.TitleExtractor.cache.Purge()
+
 	atomic.StoreInt32(&titleEnable, 1)
 	c, err := b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
 	require.NoError(t, err)
@@ -156,7 +163,7 @@ func TestService_SetTitle(t *testing.T) {
 }
 
 func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -172,11 +179,13 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, c.Score)
+	assert.Equal(t, 1, c.Vote)
 	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
 
 	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
@@ -190,6 +199,8 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 1, res[0].Score)
+	assert.Equal(t, 1, res[0].Vote)
+	assert.Equal(t, 0.0, res[0].Controversy)
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
 	assert.Nil(t, err, "vote reset")
@@ -197,11 +208,12 @@ func TestService_Vote(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "vote reset ok")
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "vote reset ok")
 }
 
 func TestService_VoteLimit(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -219,7 +231,7 @@ func TestService_VoteLimit(t *testing.T) {
 }
 
 func TestService_VotesDisabled(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -227,7 +239,7 @@ func TestService_VotesDisabled(t *testing.T) {
 }
 
 func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -243,7 +255,7 @@ func TestService_VoteAggressive(t *testing.T) {
 	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
+	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
 	// add a vote as user2
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
@@ -255,7 +267,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 		}()
 	}
 	wg.Wait()
@@ -274,7 +286,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
 		}()
 	}
 	wg.Wait()
@@ -287,7 +299,7 @@ func TestService_VoteAggressive(t *testing.T) {
 
 func TestService_VoteConcurrent(t *testing.T) {
 
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -304,21 +316,92 @@ func TestService_VoteConcurrent(t *testing.T) {
 	var wg sync.WaitGroup
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		i := i
+		ii := i
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
+				fmt.Sprintf("user1-%d", ii), true)
 		}()
 	}
 	wg.Wait()
 	res, err = b.Last("radio-t", 0)
 	require.NoError(t, err)
-	assert.Equal(t, 100, res[0].Score, "should have 1000 score")
+	assert.Equal(t, 100, res[0].Score, "should have 100 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 1000 votes")
+	assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
+	assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
+}
+
+func TestService_VotePositive(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: true}
+
+	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.EqualError(t, err, "minimal score reached for comment id-1")
+
+	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
+	assert.Nil(t, err, "minimal score doesn't affect positive vote")
+
+	b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+		MaxVotes: -1, PositiveScore: false}
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
+	assert.Nil(t, err, "minimal score ignored")
+	assert.Equal(t, -1, c.Score)
+	assert.Equal(t, 0.0, c.Controversy)
+}
+
+func TestService_VoteControversy(t *testing.T) {
+	defer teardown(t)
+	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+
+	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
+	assert.NoError(t, err)
+	assert.Equal(t, -1, c.Score, "should have -1 score")
+	assert.InDelta(t, 0.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user3", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c.Score, "should have 0 score")
+	assert.InDelta(t, 2.00, c.Controversy, 0.01)
+
+	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c.Score, "should have 1 score")
+	assert.InDelta(t, 1.73, c.Controversy, 0.01)
+
+	// check if stored
+	res, err := b.Last("radio-t", 0)
+	require.NoError(t, err)
+	assert.Equal(t, 1, res[0].Score, "should have 1 score")
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+}
+
+func TestService_Controversy(t *testing.T) {
+	tbl := []struct {
+		ups, downs int
+		res        float64
+	}{
+		{0, 0, 0},
+		{10, 5, 3.87},
+		{20, 5, 2.24},
+		{20, 50, 5.47},
+		{20, 0, 0},
+		{1100, 500, 28.60},
+		{1100, 12100, 2.37},
+		{100, 100, 200},
+		{101, 101, 202},
+	}
+
+	b := DataStore{}
+	for i, tt := range tbl {
+		t.Run(fmt.Sprintf("check-%d-%d:%d", i, tt.ups, tt.downs), func(t *testing.T) {
+			assert.InDelta(t, tt.res, b.controversy(tt.ups, tt.downs), 0.01)
+		})
+	}
 }
 
 func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -342,7 +425,7 @@ func TestService_Pin(t *testing.T) {
 }
 
 func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -369,7 +452,7 @@ func TestService_EditComment(t *testing.T) {
 }
 
 func TestService_DeleteComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -388,7 +471,7 @@ func TestService_DeleteComment(t *testing.T) {
 }
 
 func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -405,7 +488,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
 }
 
 func TestService_EditCommentReplyFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -456,7 +539,7 @@ func TestService_ValidateComment(t *testing.T) {
 }
 
 func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := prepStoreEngine(t) // two comments for https://radio-t.com
 
 	// add one more for https://radio-t.com/2
@@ -485,7 +568,7 @@ func TestService_Counts(t *testing.T) {
 }
 
 func TestService_GetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -516,7 +599,7 @@ func TestService_GetMetas(t *testing.T) {
 }
 
 func TestService_SetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -540,7 +623,7 @@ func TestService_SetMetas(t *testing.T) {
 }
 
 func TestService_IsAdmin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
@@ -550,7 +633,7 @@ func TestService_IsAdmin(t *testing.T) {
 }
 
 func TestService_HasReplies(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
@@ -579,9 +662,69 @@ func TestService_HasReplies(t *testing.T) {
 	assert.True(t, b.HasReplies(comment))
 }
 
+func TestService_Find(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "123456",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+		Score:     1,
+		Votes:     map[string]bool{"id-1": true, "id-2": true, "123456": false},
+	}
+	_, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy
+	assert.Nil(t, err)
+
+	// make sure Controversy altered
+	res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy")
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, "123456", res[0].ID)
+	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
+	assert.Equal(t, "id-1", res[1].ID)
+	assert.InDelta(t, 0, res[1].Controversy, 0.01)
+}
+
+func TestService_submitImages(t *testing.T) {
+	defer teardown(t)
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	mockStore := image.MockStore{}
+	mockStore.On("Commit", mock.Anything, mock.Anything).Times(2).Return(nil)
+	imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
+
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+
+	b.submitImages(c)
+	time.Sleep(250 * time.Millisecond)
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
+	_ = os.Remove(testDb)
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -609,3 +752,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 
 	return b
 }
+
+func teardown(_ *testing.T) {
+	_ = os.Remove(testDb)
+}

backend/app/store/service/title.go

@@ -3,19 +3,23 @@ package service
 import (
 	"io"
 	"net/http"
+	"time"
 
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 	"golang.org/x/net/html"
 )
 
-const teMaxCachedRecs = 1000
+const (
+	teCacheMaxRecs = 1000
+	teCacheTTL     = 15 * time.Minute
+)
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
 	client http.Client
-	cache  cache.LoadingCache
+	cache  lcw.LoadingCache
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
@@ -24,19 +28,19 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {
 		client: client,
 	}
 	var err error
-	res.cache, err = cache.NewMemoryCache(cache.MaxKeys(teMaxCachedRecs))
+	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
-		log.Printf("[WARN] failed to make cache, %v", err)
+		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
-		res.cache = &cache.Nop{}
+		res.cache = &lcw.Nop{}
 	}
 	return &res
 }
 
 // Get page for url and return title
 func (t *TitleExtractor) Get(url string) (string, error) {
-
+	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
-	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
+	b, err := t.cache.Get(url, func() (lcw.Value, error) {
-		resp, err := t.client.Get(url)
+		resp, err := client.Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
@@ -49,14 +53,16 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if !ok {
 			return nil, errors.Errorf("can't get title for %s", url)
 		}
-		return []byte(title), nil
+		return title, nil
 	})
 
+	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
+		_, _ = t.cache.Get(url, func() (lcw.Value, error) { return "", nil })
 		return "", err
 	}
 
-	return string(b), nil
+	return b.(string), nil
 }
 
 // get title from body reader, traverse recursively

backend/app/store/service/title_test.go

@@ -1,14 +1,17 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/syncs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +44,8 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -55,9 +59,59 @@ func TestTitle_Get(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/good")
+		r, e := ex.Get(ts.URL + "/good")
-		require.Nil(t, err)
+		require.Nil(t, e)
-		assert.Equal(t, "blah 123", title)
+		assert.Equal(t, "blah 123", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
+
+func TestTitle_GetConcurrent(t *testing.T) {
+	body := ""
+	for n := 0; n < 1000; n++ {
+		body += "something something blah blah\n"
+	}
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.String(), "/good") {
+			atomic.AddInt32(&hits, 1)
+			_, err := w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			assert.NoError(t, err)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	g := syncs.NewSizedGroup(10)
+
+	for i := 0; i < 100; i++ {
+		ii := i
+		g.Go(func(_ context.Context) {
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(ii))
+			require.Nil(t, err)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(ii), title)
+		})
+	}
+	g.Wait()
+	assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
+}
+
+func TestTitle_GetFailed(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(404)
+	}))
+
+	_, err := ex.Get(ts.URL + "/bad")
+	require.NotNil(t, err)
+
+	for i := 0; i < 100; i++ {
+		r, e := ex.Get(ts.URL + "/bad")
+		require.Nil(t, e)
+		assert.Equal(t, "", r)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
+}

backend/app/store/user.go

@@ -2,7 +2,7 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1"
+	"crypto/sha1" //nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -41,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id)
+	return hashWithFallback(sha1.New(), id) //nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -0,0 +1,41 @@
+module github.com/umputun/remark/backend
+
+go 1.12
+
+require (
+	github.com/PuerkitoBio/goquery v1.4.0
+	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/coreos/bbolt v1.3.0
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/didip/tollbooth v4.0.0+incompatible
+	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
+	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
+	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/cors v1.0.0
+	github.com/go-chi/render v1.0.0
+	github.com/go-pkgz/auth v0.5.0
+	github.com/go-pkgz/lcw v0.2.0
+	github.com/go-pkgz/lgr v0.6.2
+	github.com/go-pkgz/mongo v1.1.2
+	github.com/go-pkgz/repeater v1.1.1
+	github.com/go-pkgz/rest v1.4.0
+	github.com/go-pkgz/syncs v1.1.0
+	github.com/golang/mock v1.2.0
+	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
+	github.com/gorilla/feeds v1.1.0
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
+	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
+	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.8.1
+	github.com/rakyll/statik v0.1.3
+	github.com/rs/xid v1.2.1
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b
+	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	gopkg.in/russross/blackfriday.v2 v2.0.0
+)

backend/go.sum

@@ -0,0 +1,117 @@
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
+github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
+github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
+github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
+github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.5.0 h1:+wqppq35x83PchZNZ7SHHYLI/e8WeETFouujDLsklac=
+github.com/go-pkgz/auth v0.5.0/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
+github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
+github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
+github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
+github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
+github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
+github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
+github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
+github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
+github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
+github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
+golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
+gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=

backend/remark.rest

@@ -1,9 +1,13 @@
 
 ### find request with tree
-GET {{host}}/api/v1/find?site={{site}}&sort=-active&format=tree&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=tree&url={{url}}
 
 ### find request with plain
-GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=plain&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+
+### find request with plain
+GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### last 50 comments
 GET {{host}}/api/v1/last/50?site={{site}}
@@ -41,7 +45,7 @@ Content-Type: application/json
 }
 
 ### update comment
-PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site=remark&url=https://radio-t.com/blah1
+PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site={{site}}&url=https://radio-t.com/blah1
 Content-Type: application/json
 
 {
@@ -50,22 +54,23 @@ Content-Type: application/json
 }
 
 ### pin comment
-PUT {{host}}/api/v1/admin/pin/3665976683?site=remark&url={{url}}&pin=1
+PUT {{host}}/api/v1/admin/pin/3665976683?site={{site}}&url={{url}}&pin=1
 
 ### vote for comment
-PUT {{host}}/api/v1/vote/3665976683?site=remark&url={{url}}&vote=1
+PUT http://127.0.0.1:8080/api/v1/vote/8a8c0b80-0d0a-41c3-84ad-f4034704e827?site={{site}}&url={{url}}&vote=-1
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### get user info
 GET {{host}}/api/v1/user
 
 ### get comment by id
-GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
+GET {{host}}/api/v1/id/73e346f4-d57d-41a8-8803-6671aa187d8e?site={{site}}&url={{url}}
 
 ### get comment by id 2
-GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
+GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site={{site}}&url=https://radio-t.com/blah1
 
 ### get comment by user id
-GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
+GET {{host}}/api/v1/comments?site={{site}}&user={{user}}&limit=5
 
 ### get comment by user id2
 GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
@@ -84,7 +89,7 @@ Content-Type: application/json
 ]
 
 ### list commented posts
-GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
+GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=0
 
 ### block user
 PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -22,6 +22,7 @@ David Symonds <dsymonds@golang.org>
 Filippo Valsorda <hi@filippo.io>
 Glenn Lewis <gmlewis@google.com>
 Ingo Oeser <nightlyone@googlemail.com>
+James Hall <james.hall@shopify.com>
 Johan Euphrosine <proppy@google.com>
 Jonathan Amsterdam <jba@google.com>
 Kunpei Sakai <namusyaka@gmail.com>

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -1,4 +1,4 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
+// Copyright 2014 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
 package metadata // import "cloud.google.com/go/compute/metadata"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -31,9 +32,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"golang.org/x/net/context"
-	"golang.org/x/net/context/ctxhttp"
 )
 
 const (
@@ -64,7 +62,7 @@ var (
 )
 
 var (
-	metaClient = &http.Client{
+	defaultClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
@@ -72,15 +70,15 @@ var (
 			}).Dial,
 			ResponseHeaderTimeout: 2 * time.Second,
 		},
-	}
+	}}
-	subscribeClient = &http.Client{
+	subscribeClient = &Client{hc: &http.Client{
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
 				Timeout:   2 * time.Second,
 				KeepAlive: 30 * time.Second,
 			}).Dial,
 		},
-	}
+	}}
 )
 
 // NotDefinedError is returned when requested metadata is not defined.
@@ -95,74 +93,16 @@ func (suffix NotDefinedError) Error() string {
 	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
 }
 
-// Get returns a value from the metadata service.
+func (c *cachedValue) get(cl *Client) (v string, err error) {
-// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
-//
-// If the GCE_METADATA_HOST environment variable is not defined, a default of
-// 169.254.169.254 will be used instead.
-//
-// If the requested metadata is not defined, the returned error will
-// be of type NotDefinedError.
-func Get(suffix string) (string, error) {
-	val, _, err := getETag(metaClient, suffix)
-	return val, err
-}
-
-// getETag returns a value from the metadata service as well as the associated
-// ETag using the provided client. This func is otherwise equivalent to Get.
-func getETag(client *http.Client, suffix string) (value, etag string, err error) {
-	// Using a fixed IP makes it very difficult to spoof the metadata service in
-	// a container, which is an important use-case for local testing of cloud
-	// deployments. To enable spoofing of the metadata service, the environment
-	// variable GCE_METADATA_HOST is first inspected to decide where metadata
-	// requests shall go.
-	host := os.Getenv(metadataHostEnv)
-	if host == "" {
-		// Using 169.254.169.254 instead of "metadata" here because Go
-		// binaries built with the "netgo" tag and without cgo won't
-		// know the search suffix for "metadata" is
-		// ".google.internal", and this IP address is documented as
-		// being stable anyway.
-		host = metadataIP
-	}
-	url := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("Metadata-Flavor", "Google")
-	req.Header.Set("User-Agent", userAgent)
-	res, err := client.Do(req)
-	if err != nil {
-		return "", "", err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == http.StatusNotFound {
-		return "", "", NotDefinedError(suffix)
-	}
-	if res.StatusCode != 200 {
-		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
-	all, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", "", err
-	}
-	return string(all), res.Header.Get("Etag"), nil
-}
-
-func getTrimmed(suffix string) (s string, err error) {
-	s, err = Get(suffix)
-	s = strings.TrimSpace(s)
-	return
-}
-
-func (c *cachedValue) get() (v string, err error) {
 	defer c.mu.Unlock()
 	c.mu.Lock()
 	if c.v != "" {
 		return c.v, nil
 	}
 	if c.trim {
-		v, err = getTrimmed(c.k)
+		v, err = cl.getTrimmed(c.k)
 	} else {
-		v, err = Get(c.k)
+		v, err = cl.Get(c.k)
 	}
 	if err == nil {
 		c.v = v
@@ -201,7 +141,7 @@ func testOnGCE() bool {
 	go func() {
 		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
 		req.Header.Set("User-Agent", userAgent)
-		res, err := ctxhttp.Do(ctx, metaClient, req)
+		res, err := defaultClient.hc.Do(req.WithContext(ctx))
 		if err != nil {
 			resc <- false
 			return
@@ -266,6 +206,255 @@ func systemInfoSuggestsGCE() bool {
 	return name == "Google" || name == "Google Compute Engine"
 }
 
+// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no
+// ResponseHeaderTimeout).
+func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+	return subscribeClient.Subscribe(suffix, fn)
+}
+
+// Get calls Client.Get on the default client.
+func Get(suffix string) (string, error) { return defaultClient.Get(suffix) }
+
+// ProjectID returns the current instance's project ID string.
+func ProjectID() (string, error) { return defaultClient.ProjectID() }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() }
+
+// InternalIP returns the instance's primary internal IP address.
+func InternalIP() (string, error) { return defaultClient.InternalIP() }
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func Hostname() (string, error) { return defaultClient.Hostname() }
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() }
+
+// InstanceID returns the current VM's numeric instance ID.
+func InstanceID() (string, error) { return defaultClient.InstanceID() }
+
+// InstanceName returns the current VM's instance ID string.
+func InstanceName() (string, error) { return defaultClient.InstanceName() }
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func Zone() (string, error) { return defaultClient.Zone() }
+
+// InstanceAttributes calls Client.InstanceAttributes on the default client.
+func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() }
+
+// ProjectAttributes calls Client.ProjectAttributes on the default client.
+func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() }
+
+// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client.
+func InstanceAttributeValue(attr string) (string, error) {
+	return defaultClient.InstanceAttributeValue(attr)
+}
+
+// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client.
+func ProjectAttributeValue(attr string) (string, error) {
+	return defaultClient.ProjectAttributeValue(attr)
+}
+
+// Scopes calls Client.Scopes on the default client.
+func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) }
+
+func strsContains(ss []string, s string) bool {
+	for _, v := range ss {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// A Client provides metadata.
+type Client struct {
+	hc *http.Client
+}
+
+// NewClient returns a Client that can be used to fetch metadata. All HTTP requests
+// will use the given http.Client instead of the default client.
+func NewClient(c *http.Client) *Client {
+	return &Client{hc: c}
+}
+
+// getETag returns a value from the metadata service as well as the associated ETag.
+// This func is otherwise equivalent to Get.
+func (c *Client) getETag(suffix string) (value, etag string, err error) {
+	// Using a fixed IP makes it very difficult to spoof the metadata service in
+	// a container, which is an important use-case for local testing of cloud
+	// deployments. To enable spoofing of the metadata service, the environment
+	// variable GCE_METADATA_HOST is first inspected to decide where metadata
+	// requests shall go.
+	host := os.Getenv(metadataHostEnv)
+	if host == "" {
+		// Using 169.254.169.254 instead of "metadata" here because Go
+		// binaries built with the "netgo" tag and without cgo won't
+		// know the search suffix for "metadata" is
+		// ".google.internal", and this IP address is documented as
+		// being stable anyway.
+		host = metadataIP
+	}
+	url := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Metadata-Flavor", "Google")
+	req.Header.Set("User-Agent", userAgent)
+	res, err := c.hc.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return "", "", NotDefinedError(suffix)
+	}
+	if res.StatusCode != 200 {
+		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
+	}
+	all, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", err
+	}
+	return string(all), res.Header.Get("Etag"), nil
+}
+
+// Get returns a value from the metadata service.
+// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
+//
+// If the GCE_METADATA_HOST environment variable is not defined, a default of
+// 169.254.169.254 will be used instead.
+//
+// If the requested metadata is not defined, the returned error will
+// be of type NotDefinedError.
+func (c *Client) Get(suffix string) (string, error) {
+	val, _, err := c.getETag(suffix)
+	return val, err
+}
+
+func (c *Client) getTrimmed(suffix string) (s string, err error) {
+	s, err = c.Get(suffix)
+	s = strings.TrimSpace(s)
+	return
+}
+
+func (c *Client) lines(suffix string) ([]string, error) {
+	j, err := c.Get(suffix)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.Split(strings.TrimSpace(j), "\n")
+	for i := range s {
+		s[i] = strings.TrimSpace(s[i])
+	}
+	return s, nil
+}
+
+// ProjectID returns the current instance's project ID string.
+func (c *Client) ProjectID() (string, error) { return projID.get(c) }
+
+// NumericProjectID returns the current instance's numeric project ID.
+func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) }
+
+// InstanceID returns the current VM's numeric instance ID.
+func (c *Client) InstanceID() (string, error) { return instID.get(c) }
+
+// InternalIP returns the instance's primary internal IP address.
+func (c *Client) InternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/ip")
+}
+
+// ExternalIP returns the instance's primary external (public) IP address.
+func (c *Client) ExternalIP() (string, error) {
+	return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
+}
+
+// Hostname returns the instance's hostname. This will be of the form
+// "<instanceID>.c.<projID>.internal".
+func (c *Client) Hostname() (string, error) {
+	return c.getTrimmed("instance/hostname")
+}
+
+// InstanceTags returns the list of user-defined instance tags,
+// assigned when initially creating a GCE instance.
+func (c *Client) InstanceTags() ([]string, error) {
+	var s []string
+	j, err := c.Get("instance/tags")
+	if err != nil {
+		return nil, err
+	}
+	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// InstanceName returns the current VM's instance ID string.
+func (c *Client) InstanceName() (string, error) {
+	host, err := c.Hostname()
+	if err != nil {
+		return "", err
+	}
+	return strings.Split(host, ".")[0], nil
+}
+
+// Zone returns the current VM's zone, such as "us-central1-b".
+func (c *Client) Zone() (string, error) {
+	zone, err := c.getTrimmed("instance/zone")
+	// zone is of the form "projects/<projNum>/zones/<zoneName>".
+	if err != nil {
+		return "", err
+	}
+	return zone[strings.LastIndex(zone, "/")+1:], nil
+}
+
+// InstanceAttributes returns the list of user-defined attributes,
+// assigned when initially creating a GCE VM instance. The value of an
+// attribute can be obtained with InstanceAttributeValue.
+func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") }
+
+// ProjectAttributes returns the list of user-defined attributes
+// applying to the project as a whole, not just this VM.  The value of
+// an attribute can be obtained with ProjectAttributeValue.
+func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") }
+
+// InstanceAttributeValue returns the value of the provided VM
+// instance attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// InstanceAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) InstanceAttributeValue(attr string) (string, error) {
+	return c.Get("instance/attributes/" + attr)
+}
+
+// ProjectAttributeValue returns the value of the provided
+// project attribute.
+//
+// If the requested attribute is not defined, the returned error will
+// be of type NotDefinedError.
+//
+// ProjectAttributeValue may return ("", nil) if the attribute was
+// defined to be the empty string.
+func (c *Client) ProjectAttributeValue(attr string) (string, error) {
+	return c.Get("project/attributes/" + attr)
+}
+
+// Scopes returns the service account scopes for the given account.
+// The account may be empty or the string "default" to use the instance's
+// main account.
+func (c *Client) Scopes(serviceAccount string) ([]string, error) {
+	if serviceAccount == "" {
+		serviceAccount = "default"
+	}
+	return c.lines("instance/service-accounts/" + serviceAccount + "/scopes")
+}
+
 // Subscribe subscribes to a value from the metadata service.
 // The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
 // The suffix may contain query parameters.
@@ -275,11 +464,11 @@ func systemInfoSuggestsGCE() bool {
 // and ok false. Subscribe blocks until fn returns a non-nil error or the value
 // is deleted. Subscribe returns the error value returned from the last call to
 // fn, which may be nil when ok == false.
-func Subscribe(suffix string, fn func(v string, ok bool) error) error {
+func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error {
 	const failedSubscribeSleep = time.Second * 5
 
 	// First check to see if the metadata value exists at all.
-	val, lastETag, err := getETag(subscribeClient, suffix)
+	val, lastETag, err := c.getETag(suffix)
 	if err != nil {
 		return err
 	}
@@ -295,7 +484,7 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 		suffix += "?wait_for_change=true&last_etag="
 	}
 	for {
-		val, etag, err := getETag(subscribeClient, suffix+url.QueryEscape(lastETag))
+		val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag))
 		if err != nil {
 			if _, deleted := err.(NotDefinedError); !deleted {
 				time.Sleep(failedSubscribeSleep)
@@ -310,128 +499,3 @@ func Subscribe(suffix string, fn func(v string, ok bool) error) error {
 		}
 	}
 }
-
-// ProjectID returns the current instance's project ID string.
-func ProjectID() (string, error) { return projID.get() }
-
-// NumericProjectID returns the current instance's numeric project ID.
-func NumericProjectID() (string, error) { return projNum.get() }
-
-// InternalIP returns the instance's primary internal IP address.
-func InternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/ip")
-}
-
-// ExternalIP returns the instance's primary external (public) IP address.
-func ExternalIP() (string, error) {
-	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
-}
-
-// Hostname returns the instance's hostname. This will be of the form
-// "<instanceID>.c.<projID>.internal".
-func Hostname() (string, error) {
-	return getTrimmed("instance/hostname")
-}
-
-// InstanceTags returns the list of user-defined instance tags,
-// assigned when initially creating a GCE instance.
-func InstanceTags() ([]string, error) {
-	var s []string
-	j, err := Get("instance/tags")
-	if err != nil {
-		return nil, err
-	}
-	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
-		return nil, err
-	}
-	return s, nil
-}
-
-// InstanceID returns the current VM's numeric instance ID.
-func InstanceID() (string, error) {
-	return instID.get()
-}
-
-// InstanceName returns the current VM's instance ID string.
-func InstanceName() (string, error) {
-	host, err := Hostname()
-	if err != nil {
-		return "", err
-	}
-	return strings.Split(host, ".")[0], nil
-}
-
-// Zone returns the current VM's zone, such as "us-central1-b".
-func Zone() (string, error) {
-	zone, err := getTrimmed("instance/zone")
-	// zone is of the form "projects/<projNum>/zones/<zoneName>".
-	if err != nil {
-		return "", err
-	}
-	return zone[strings.LastIndex(zone, "/")+1:], nil
-}
-
-// InstanceAttributes returns the list of user-defined attributes,
-// assigned when initially creating a GCE VM instance. The value of an
-// attribute can be obtained with InstanceAttributeValue.
-func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }
-
-// ProjectAttributes returns the list of user-defined attributes
-// applying to the project as a whole, not just this VM.  The value of
-// an attribute can be obtained with ProjectAttributeValue.
-func ProjectAttributes() ([]string, error) { return lines("project/attributes/") }
-
-func lines(suffix string) ([]string, error) {
-	j, err := Get(suffix)
-	if err != nil {
-		return nil, err
-	}
-	s := strings.Split(strings.TrimSpace(j), "\n")
-	for i := range s {
-		s[i] = strings.TrimSpace(s[i])
-	}
-	return s, nil
-}
-
-// InstanceAttributeValue returns the value of the provided VM
-// instance attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// InstanceAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func InstanceAttributeValue(attr string) (string, error) {
-	return Get("instance/attributes/" + attr)
-}
-
-// ProjectAttributeValue returns the value of the provided
-// project attribute.
-//
-// If the requested attribute is not defined, the returned error will
-// be of type NotDefinedError.
-//
-// ProjectAttributeValue may return ("", nil) if the attribute was
-// defined to be the empty string.
-func ProjectAttributeValue(attr string) (string, error) {
-	return Get("project/attributes/" + attr)
-}
-
-// Scopes returns the service account scopes for the given account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
-func Scopes(serviceAccount string) ([]string, error) {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return lines("instance/service-accounts/" + serviceAccount + "/scopes")
-}
-
-func strsContains(ss []string, s string) bool {
-	for _, v := range ss {
-		if v == s {
-			return true
-		}
-	}
-	return false
-}

backend/vendor/github.com/davecgh/go-spew/LICENSE

@@ -2,7 +2,7 @@ ISC License
 
 Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
 
-Permission to use, copy, modify, and distribute this software for any
+Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
 copyright notice and this permission notice appear in all copies.
 

backend/vendor/github.com/davecgh/go-spew/spew/bypass.go

@@ -16,7 +16,9 @@
 // when the code is not running on Google App Engine, compiled by GopherJS, and
 // "-tags safe" is not added to the go build command line.  The "disableunsafe"
 // tag is deprecated and thus should not be used.
-// +build !js,!appengine,!safe,!disableunsafe
+// Go versions prior to 1.4 are disabled because they use a different layout
+// for interfaces which make the implementation of unsafeReflectValue more complex.
+// +build !js,!appengine,!safe,!disableunsafe,go1.4
 
 package spew
 
@@ -34,80 +36,49 @@ const (
 	ptrSize = unsafe.Sizeof((*byte)(nil))
 )
 
-var (
+type flag uintptr
-	// offsetPtr, offsetScalar, and offsetFlag are the offsets for the
-	// internal reflect.Value fields.  These values are valid before golang
-	// commit ecccf07e7f9d which changed the format.  The are also valid
-	// after commit 82f48826c6c7 which changed the format again to mirror
-	// the original format.  Code in the init function updates these offsets
-	// as necessary.
-	offsetPtr    = uintptr(ptrSize)
-	offsetScalar = uintptr(0)
-	offsetFlag   = uintptr(ptrSize * 2)
 
-	// flagKindWidth and flagKindShift indicate various bits that the
+var (
-	// reflect package uses internally to track kind information.
+	// flagRO indicates whether the value field of a reflect.Value
-	//
+	// is read-only.
-	// flagRO indicates whether or not the value field of a reflect.Value is
+	flagRO flag
-	// read-only.
+
-	//
+	// flagAddr indicates whether the address of the reflect.Value's
-	// flagIndir indicates whether the value field of a reflect.Value is
+	// value may be taken.
-	// the actual data or a pointer to the data.
+	flagAddr flag
-	//
-	// These values are valid before golang commit 90a7c3c86944 which
-	// changed their positions.  Code in the init function updates these
-	// flags as necessary.
-	flagKindWidth = uintptr(5)
-	flagKindShift = uintptr(flagKindWidth - 1)
-	flagRO        = uintptr(1 << 0)
-	flagIndir     = uintptr(1 << 1)
 )
 
-func init() {
+// flagKindMask holds the bits that make up the kind
-	// Older versions of reflect.Value stored small integers directly in the
+// part of the flags field. In all the supported versions,
-	// ptr field (which is named val in the older versions).  Versions
+// it is in the lower 5 bits.
-	// between commits ecccf07e7f9d and 82f48826c6c7 added a new field named
+const flagKindMask = flag(0x1f)
-	// scalar for this purpose which unfortunately came before the flag
-	// field, so the offset of the flag field is different for those
-	// versions.
-	//
-	// This code constructs a new reflect.Value from a known small integer
-	// and checks if the size of the reflect.Value struct indicates it has
-	// the scalar field. When it does, the offsets are updated accordingly.
-	vv := reflect.ValueOf(0xf00)
-	if unsafe.Sizeof(vv) == (ptrSize * 4) {
-		offsetScalar = ptrSize * 2
-		offsetFlag = ptrSize * 3
-	}
 
-	// Commit 90a7c3c86944 changed the flag positions such that the low
+// Different versions of Go have used different
-	// order bits are the kind.  This code extracts the kind from the flags
+// bit layouts for the flags type. This table
-	// field and ensures it's the correct type.  When it's not, the flag
+// records the known combinations.
-	// order has been changed to the newer format, so the flags are updated
+var okFlags = []struct {
-	// accordingly.
+	ro, addr flag
-	upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag)
+}{{
-	upfv := *(*uintptr)(upf)
+	// From Go 1.4 to 1.5
-	flagKindMask := uintptr((1<<flagKindWidth - 1) << flagKindShift)
+	ro:   1 << 5,
-	if (upfv&flagKindMask)>>flagKindShift != uintptr(reflect.Int) {
+	addr: 1 << 7,
-		flagKindShift = 0
+}, {
-		flagRO = 1 << 5
+	// Up to Go tip.
-		flagIndir = 1 << 6
+	ro:   1<<5 | 1<<6,
+	addr: 1 << 8,
+}}
 
-		// Commit adf9b30e5594 modified the flags to separate the
+var flagValOffset = func() uintptr {
-		// flagRO flag into two bits which specifies whether or not the
+	field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
-		// field is embedded.  This causes flagIndir to move over a bit
+	if !ok {
-		// and means that flagRO is the combination of either of the
+		panic("reflect.Value has no flag field")
-		// original flagRO bit and the new bit.
-		//
-		// This code detects the change by extracting what used to be
-		// the indirect bit to ensure it's set.  When it's not, the flag
-		// order has been changed to the newer format, so the flags are
-		// updated accordingly.
-		if upfv&flagIndir == 0 {
-			flagRO = 3 << 5
-			flagIndir = 1 << 7
-		}
 	}
+	return field.Offset
+}()
+
+// flagField returns a pointer to the flag field of a reflect.Value.
+func flagField(v *reflect.Value) *flag {
+	return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
 }
 
 // unsafeReflectValue converts the passed reflect.Value into a one that bypasses
@@ -119,34 +90,56 @@ func init() {
 // This allows us to check for implementations of the Stringer and error
 // interfaces to be used for pretty printing ordinarily unaddressable and
 // inaccessible values such as unexported struct fields.
-func unsafeReflectValue(v reflect.Value) (rv reflect.Value) {
+func unsafeReflectValue(v reflect.Value) reflect.Value {
-	indirects := 1
+	if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
-	vt := v.Type()
+		return v
-	upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr)
+	}
-	rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag))
+	flagFieldPtr := flagField(&v)
-	if rvf&flagIndir != 0 {
+	*flagFieldPtr &^= flagRO
-		vt = reflect.PtrTo(v.Type())
+	*flagFieldPtr |= flagAddr
-		indirects++
+	return v
-	} else if offsetScalar != 0 {
+}
-		// The value is in the scalar field when it's not one of the
+
-		// reference types.
+// Sanity checks against future reflect package changes
-		switch vt.Kind() {
+// to the type or semantics of the Value.flag field.
-		case reflect.Uintptr:
+func init() {
-		case reflect.Chan:
+	field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
-		case reflect.Func:
+	if !ok {
-		case reflect.Map:
+		panic("reflect.Value has no flag field")
-		case reflect.Ptr:
+	}
-		case reflect.UnsafePointer:
+	if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
-		default:
+		panic("reflect.Value flag field has changed kind")
-			upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) +
+	}
-				offsetScalar)
+	type t0 int
+	var t struct {
+		A t0
+		// t0 will have flagEmbedRO set.
+		t0
+		// a will have flagStickyRO set
+		a t0
+	}
+	vA := reflect.ValueOf(t).FieldByName("A")
+	va := reflect.ValueOf(t).FieldByName("a")
+	vt0 := reflect.ValueOf(t).FieldByName("t0")
+
+	// Infer flagRO from the difference between the flags
+	// for the (otherwise identical) fields in t.
+	flagPublic := *flagField(&vA)
+	flagWithRO := *flagField(&va) | *flagField(&vt0)
+	flagRO = flagPublic ^ flagWithRO
+
+	// Infer flagAddr from the difference between a value
+	// taken from a pointer and not.
+	vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
+	flagNoPtr := *flagField(&vA)
+	flagPtr := *flagField(&vPtrA)
+	flagAddr = flagNoPtr ^ flagPtr
+
+	// Check that the inferred flags tally with one of the known versions.
+	for _, f := range okFlags {
+		if flagRO == f.ro && flagAddr == f.addr {
+			return
 		}
 	}
-
+	panic("reflect.Value read-only flag has changed semantics")
-	pv := reflect.NewAt(vt, upv)
-	rv = pv
-	for i := 0; i < indirects; i++ {
-		rv = rv.Elem()
-	}
-	return rv
 }

backend/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go

@@ -16,7 +16,7 @@
 // when the code is running on Google App Engine, compiled by GopherJS, or
 // "-tags safe" is added to the go build command line.  The "disableunsafe"
 // tag is deprecated and thus should not be used.
-// +build js appengine safe disableunsafe
+// +build js appengine safe disableunsafe !go1.4
 
 package spew
 

backend/vendor/github.com/davecgh/go-spew/spew/common.go

@@ -180,7 +180,7 @@ func printComplex(w io.Writer, c complex128, floatPrecision int) {
 	w.Write(closeParenBytes)
 }
 
-// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x'
+// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x'
 // prefix to Writer w.
 func printHexPtr(w io.Writer, p uintptr) {
 	// Null pointer.

backend/vendor/github.com/davecgh/go-spew/spew/dump.go

@@ -35,16 +35,16 @@ var (
 
 	// cCharRE is a regular expression that matches a cgo char.
 	// It is used to detect character arrays to hexdump them.
-	cCharRE = regexp.MustCompile("^.*\\._Ctype_char$")
+	cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`)
 
 	// cUnsignedCharRE is a regular expression that matches a cgo unsigned
 	// char.  It is used to detect unsigned character arrays to hexdump
 	// them.
-	cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$")
+	cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`)
 
 	// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
 	// It is used to detect uint8_t arrays to hexdump them.
-	cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$")
+	cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`)
 )
 
 // dumpState contains information about the state of a dump operation.
@@ -143,10 +143,10 @@ func (d *dumpState) dumpPtr(v reflect.Value) {
 	// Display dereferenced value.
 	d.w.Write(openParenBytes)
 	switch {
-	case nilFound == true:
+	case nilFound:
 		d.w.Write(nilAngleBytes)
 
-	case cycleFound == true:
+	case cycleFound:
 		d.w.Write(circularBytes)
 
 	default:

backend/vendor/github.com/davecgh/go-spew/spew/format.go

@@ -182,10 +182,10 @@ func (f *formatState) formatPtr(v reflect.Value) {
 
 	// Display dereferenced value.
 	switch {
-	case nilFound == true:
+	case nilFound:
 		f.fs.Write(nilAngleBytes)
 
-	case cycleFound == true:
+	case cycleFound:
 		f.fs.Write(circularShortBytes)
 
 	default:

backend/vendor/github.com/globalsign/mgo/.travis.yml

@@ -29,7 +29,7 @@ install:
     - go get gopkg.in/check.v1
     - go get gopkg.in/yaml.v2
     - go get gopkg.in/tomb.v2
-    - go get github.com/golang/lint
+    - go get golang.org/x/lint/golint
 
 before_script:
     - golint  ./... | grep -v 'ID' | cat

backend/vendor/github.com/globalsign/mgo/README.md

@@ -13,6 +13,12 @@ Detailed documentation of the API is available at
 
 A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver.
 
+## Supported Versions
+
+`mgo` is known to work well on (and has integration tests against) MongoDB v3.0, 3.2, 3.4 and 3.6. 
+
+MongoDB 4.0 is currently experimental - we would happily accept PRs to help improve support!
+
 ## Changes
 * Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254))
 * Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288))

backend/vendor/github.com/go-chi/chi/.travis.yml

@@ -1,18 +1,18 @@
 language: go
 
 go:
-  - 1.7.x
+  - 1.10.x
-  - 1.8.x
+  - 1.11.x
-  - 1.9.x
+  - 1.12.x
-
-install:
-  - go get -u golang.org/x/tools/cmd/goimports
-  - go get -u github.com/golang/lint/golint
 
 script:
   - go get -d -t ./...
   - go vet ./...
-  - golint ./...
   - go test ./...
   - >
-    goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :
+    go_version=$(go version);
+    if [ ${go_version:13:4} = "1.12" ]; then
+      go get -u golang.org/x/tools/cmd/goimports;
+      goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :;
+    fi
+

backend/vendor/github.com/go-chi/chi/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v4.0.0 (2019-01-10)
+
+- chi v4 requires Go 1.10.3+ (or Go 1.9.7+) - we have deprecated support for Go 1.7 and 1.8
+- router: respond with 404 on router with no routes (#362)
+- router: additional check to ensure wildcard is at the end of a url pattern (#333)
+- middleware: deprecate use of http.CloseNotifier (#347)
+- middleware: fix RedirectSlashes to include query params on redirect (#334)
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.4...v4.0.0
+
+
+## v3.3.4 (2019-01-07)
+
+- Minor middleware improvements. No changes to core library/router. Moving v3 into its
+- own branch as a version of chi for Go 1.7, 1.8, 1.9, 1.10, 1.11
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.3...v3.3.4
+
+
+## v3.3.3 (2018-08-27)
+
+- Minor release
+- See https://github.com/go-chi/chi/compare/v3.3.2...v3.3.3
+
+
 ## v3.3.2 (2017-12-22)
 
 - Support to route trailing slashes on mounted sub-routers (#281)

backend/vendor/github.com/go-chi/chi/README.md

@@ -3,7 +3,7 @@
 
 [![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis]
 
-`chi` is a lightweight, idiomatic and composable router for building Go 1.7+ HTTP services. It's
+`chi` is a lightweight, idiomatic and composable router for building Go HTTP services. It's
 especially good at helping you write large REST API services that are kept maintainable as your
 project grows and changes. `chi` is built on the new `context` package introduced in Go 1.7 to
 handle signaling, cancelation and request-scoped values across a handler chain.
@@ -31,18 +31,12 @@ included some useful/optional subpackages: [middleware](/middleware), [render](h
 * **Context control** - built on new `context` package, providing value chaining, cancelations and timeouts
 * **Robust** - in production at Pressly, CloudFlare, Heroku, 99Designs, and many others (see [discussion](https://github.com/go-chi/chi/issues/91))
 * **Doc generation** - `docgen` auto-generates routing documentation from your source to JSON or Markdown
-* **No external dependencies** - plain ol' Go 1.7+ stdlib + net/http
+* **No external dependencies** - plain ol' Go stdlib + net/http
 
 
 ## Examples
 
-* [rest](https://github.com/go-chi/chi/blob/master/_examples/rest/main.go) - REST APIs made easy, productive and maintainable
+See [_examples/](https://github.com/go-chi/chi/blob/master/_examples/) for a variety of examples.
-* [logging](https://github.com/go-chi/chi/blob/master/_examples/logging/main.go) - Easy structured logging for any backend
-* [limits](https://github.com/go-chi/chi/blob/master/_examples/limits/main.go) - Timeouts and Throttling
-* [todos-resource](https://github.com/go-chi/chi/blob/master/_examples/todos-resource/main.go) - Struct routers/handlers, an example of another code layout style
-* [versions](https://github.com/go-chi/chi/blob/master/_examples/versions/main.go) - Demo of `chi/render` subpkg
-* [fileserver](https://github.com/go-chi/chi/blob/master/_examples/fileserver/main.go) - Easily serve static files
-* [graceful](https://github.com/go-chi/chi/blob/master/_examples/graceful/main.go) - Graceful context signaling and server shutdown
 
 
 **As easy as:**
@@ -70,8 +64,8 @@ Here is a little preview of how routing looks like with chi. Also take a look at
 in JSON ([routes.json](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.json)) and in
 Markdown ([routes.md](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.md)).
 
-I highly recommend reading the source of the [examples](#examples) listed above, they will show you all the features
+I highly recommend reading the source of the [examples](https://github.com/go-chi/chi/blob/master/_examples/) listed
-of chi and serve as a good form of documentation.
+above, they will show you all the features of chi and serve as a good form of documentation.
 
 ```go
 import (
@@ -232,7 +226,7 @@ type Router interface {
 }
 
 // Routes interface adds two methods for router traversal, which is also
-// used by the `docgen` subpackage to generation documentation for Routers.
+// used by the github.com/go-chi/docgen package to generate documentation for Routers.
 type Routes interface {
 	// Routes returns the routing tree in an easily traversable structure.
 	Routes() []Route
@@ -261,7 +255,7 @@ friendly with any middleware in the community. This offers much better extensibi
 of packages and is at the heart of chi's purpose.
 
 Here is an example of a standard net/http middleware handler using the new request context
-available in Go 1.7+. This middleware sets a hypothetical user identifier on the request
+available in Go. This middleware sets a hypothetical user identifier on the request
 context and calls the next handler in the chain.
 
 ```go
@@ -347,6 +341,7 @@ Please see https://github.com/go-chi for additional packages.
 | package                                            | description                                                 |
 |:---------------------------------------------------|:-------------------------------------------------------------
 | [cors](https://github.com/go-chi/cors)             | Cross-origin resource sharing (CORS)                        |
+| [docgen](https://github.com/go-chi/docgen)         | Print chi.Router routes at runtime                          |
 | [jwtauth](https://github.com/go-chi/jwtauth)       | JWT authentication                                          |
 | [hostrouter](https://github.com/go-chi/hostrouter) | Domain/host based request routing                           |
 | [httpcoala](https://github.com/go-chi/httpcoala)   | HTTP request coalescer                                      |
@@ -374,33 +369,33 @@ and..
 
 The benchmark suite: https://github.com/pkieltyka/go-http-routing-benchmark
 
-Results as of Aug 31, 2017 on Go 1.9.0
+Results as of Jan 9, 2019 with Go 1.11.4 on Linux X1 Carbon laptop
 
 ```shell
-BenchmarkChi_Param        	 3000000	       607 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_Param            3000000         475 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_Param5       	 2000000	       935 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_Param5           2000000         696 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_Param20      	 1000000	      1944 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_Param20          1000000        1275 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_ParamWrite   	 2000000	       664 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_ParamWrite       3000000         505 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GithubStatic 	 2000000	       627 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_GithubStatic     3000000         508 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GithubParam  	 2000000	       847 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_GithubParam      2000000         669 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GithubAll    	   10000	    175556 ns/op	   87700 B/op	     609 allocs/op
+BenchmarkChi_GithubAll          10000      134627 ns/op     87699 B/op    609 allocs/op
-BenchmarkChi_GPlusStatic  	 3000000	       566 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_GPlusStatic      3000000         402 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GPlusParam   	 2000000	       652 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_GPlusParam       3000000         500 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GPlus2Params 	 2000000	       767 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_GPlus2Params     3000000         586 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_GPlusAll     	  200000	      9794 ns/op	    5616 B/op	      39 allocs/op
+BenchmarkChi_GPlusAll          200000        7237 ns/op      5616 B/op     39 allocs/op
-BenchmarkChi_ParseStatic  	 3000000	       590 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_ParseStatic      3000000         408 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_ParseParam   	 2000000	       656 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_ParseParam       3000000         488 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_Parse2Params 	 2000000	       715 ns/op	     432 B/op	       3 allocs/op
+BenchmarkChi_Parse2Params     3000000         551 ns/op       432 B/op      3 allocs/op
-BenchmarkChi_ParseAll     	  100000	     18045 ns/op	   11232 B/op	      78 allocs/op
+BenchmarkChi_ParseAll          100000       13508 ns/op     11232 B/op     78 allocs/op
-BenchmarkChi_StaticAll    	   10000	    108871 ns/op	   67827 B/op	     471 allocs/op
+BenchmarkChi_StaticAll          20000       81933 ns/op     67826 B/op    471 allocs/op
 ```
 
-Comparison with other routers: https://gist.github.com/pkieltyka/c089f309abeb179cfc4deaa519956d8c
+Comparison with other routers: https://gist.github.com/pkieltyka/123032f12052520aaccab752bd3e78cc
 
 NOTE: the allocs in the benchmark above are from the calls to http.Request's
 `WithContext(context.Context)` method that clones the http.Request, sets the `Context()`
 on the duplicated (alloc'd) request and returns it the new request object. This is just
-how setting context on a request in Go 1.7+ works.
+how setting context on a request in Go works.
 
 
 ## Credits

backend/vendor/github.com/go-chi/chi/context.go

@@ -84,13 +84,13 @@ func (x *Context) URLParam(key string) string {
 //
 // For example,
 //
-// func Instrument(next http.Handler) http.Handler {
+//   func Instrument(next http.Handler) http.Handler {
-//   return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+//     return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-//     next.ServeHTTP(w, r)
+//       next.ServeHTTP(w, r)
-//     routePattern := chi.RouteContext(r.Context()).RoutePattern()
+//       routePattern := chi.RouteContext(r.Context()).RoutePattern()
-//     measure(w, r, routePattern)
+//       measure(w, r, routePattern)
-// 	 })
+//   	 })
-// }
+//   }
 func (x *Context) RoutePattern() string {
 	routePattern := strings.Join(x.RoutePatterns, "")
 	return strings.Replace(routePattern, "/*/", "/", -1)

backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go

@@ -1,42 +0,0 @@
-// +build go1.7,!go1.8
-
-package middleware
-
-import (
-	"context"
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		cn, ok := w.(http.CloseNotifier)
-		if !ok {
-			panic("chi/middleware: CloseNotify expects http.ResponseWriter to implement http.CloseNotifier interface")
-		}
-		closeNotifyCh := cn.CloseNotify()
-
-		ctx, cancel := context.WithCancel(r.Context())
-		defer cancel()
-
-		go func() {
-			select {
-			case <-ctx.Done():
-				return
-			case <-closeNotifyCh:
-				cancel()
-				return
-			}
-		}()
-
-		r = r.WithContext(ctx)
-		next.ServeHTTP(w, r)
-	}
-
-	return http.HandlerFunc(fn)
-}

backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go

@@ -1,17 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	return next
-}

backend/vendor/github.com/go-chi/chi/middleware/compress.go

@@ -11,24 +11,98 @@ import (
 	"strings"
 )
 
-type encoding int
+var encoders = map[string]EncoderFunc{}
 
-const (
+var encodingPrecedence = []string{"br", "gzip", "deflate"}
-	encodingNone encoding = iota
+
-	encodingGzip
+func init() {
-	encodingDeflate
+	// TODO:
-)
+	// lzma: Opera.
+	// sdch: Chrome, Android. Gzip output + dictionary header.
+	// br:   Brotli, see https://github.com/go-chi/chi/pull/326
+
+	// TODO: Exception for old MSIE browsers that can't handle non-HTML?
+	// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	SetEncoder("gzip", encoderGzip)
+
+	// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
+	// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
+	// checksum compared to CRC-32 used in "gzip" and thus is faster.
+	//
+	// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
+	// raw DEFLATE data only, without the mentioned zlib wrapper.
+	// Because of this major confusion, most modern browsers try it
+	// both ways, first looking for zlib headers.
+	// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
+	//
+	// The list of browsers having problems is quite big, see:
+	// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
+	//
+	// That's why we prefer gzip over deflate. It's just more reliable
+	// and not significantly slower than gzip.
+	SetEncoder("deflate", encoderDeflate)
+
+	// NOTE: Not implemented, intentionally:
+	// case "compress": // LZW. Deprecated.
+	// case "bzip2":    // Too slow on-the-fly.
+	// case "zopfli":   // Too slow on-the-fly.
+	// case "xz":       // Too slow on-the-fly.
+}
+
+// An EncoderFunc is a function that wraps the provided ResponseWriter with a
+// streaming compression algorithm and returns it.
+//
+// In case of failure, the function should return nil.
+type EncoderFunc func(w http.ResponseWriter, level int) io.Writer
+
+// SetEncoder can be used to set the implementation of a compression algorithm.
+//
+// The encoding should be a standardised identifier. See:
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
+//
+// For example, add the Brotli algortithm:
+//
+//  import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc"
+//
+//  middleware.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer {
+//    params := brotli_enc.NewBrotliParams()
+//    params.SetQuality(level)
+//    return brotli_enc.NewBrotliWriter(params, w)
+//  })
+func SetEncoder(encoding string, fn EncoderFunc) {
+	encoding = strings.ToLower(encoding)
+	if encoding == "" {
+		panic("the encoding can not be empty")
+	}
+	if fn == nil {
+		panic("attempted to set a nil encoder function")
+	}
+	encoders[encoding] = fn
+
+	var e string
+	for _, v := range encodingPrecedence {
+		if v == encoding {
+			e = v
+		}
+	}
+
+	if e == "" {
+		encodingPrecedence = append([]string{e}, encodingPrecedence...)
+	}
+}
 
 var defaultContentTypes = map[string]struct{}{
-	"text/html":                struct{}{},
+	"text/html":                {},
-	"text/css":                 struct{}{},
+	"text/css":                 {},
-	"text/plain":               struct{}{},
+	"text/plain":               {},
-	"text/javascript":          struct{}{},
+	"text/javascript":          {},
-	"application/javascript":   struct{}{},
+	"application/javascript":   {},
-	"application/x-javascript": struct{}{},
+	"application/x-javascript": {},
-	"application/json":         struct{}{},
+	"application/json":         {},
-	"application/atom+xml":     struct{}{},
+	"application/atom+xml":     {},
-	"application/rss+xml":      struct{}{},
+	"application/rss+xml":      {},
+	"image/svg+xml":            {},
 }
 
 // DefaultCompress is a middleware that compresses response
@@ -43,6 +117,11 @@ func DefaultCompress(next http.Handler) http.Handler {
 // body of a given content types to a data format based
 // on Accept-Encoding request header. It uses a given
 // compression level.
+//
+// NOTE: make sure to set the Content-Type header on your response
+// otherwise this middleware will not compress the response body. For ex, in
+// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody))
+// or set it manually.
 func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 	contentTypes := defaultContentTypes
 	if len(types) > 0 {
@@ -54,159 +133,143 @@ func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
-			mcw := &maybeCompressResponseWriter{
+			encoder, encoding := selectEncoder(r.Header)
+
+			cw := &compressResponseWriter{
 				ResponseWriter: w,
 				w:              w,
 				contentTypes:   contentTypes,
-				encoding:       selectEncoding(r.Header),
+				encoder:        encoder,
+				encoding:       encoding,
 				level:          level,
 			}
-			defer mcw.Close()
+			defer cw.Close()
 
-			next.ServeHTTP(mcw, r)
+			next.ServeHTTP(cw, r)
 		}
 
 		return http.HandlerFunc(fn)
 	}
 }
 
-func selectEncoding(h http.Header) encoding {
+func selectEncoder(h http.Header) (EncoderFunc, string) {
-	enc := h.Get("Accept-Encoding")
+	header := h.Get("Accept-Encoding")
 
-	switch {
+	// Parse the names of all accepted algorithms from the header.
-	// TODO:
+	accepted := strings.Split(strings.ToLower(header), ",")
-	// case "br":    // Brotli, experimental. Firefox 2016, to-be-in Chromium.
-	// case "lzma":  // Opera.
-	// case "sdch":  // Chrome, Android. Gzip output + dictionary header.
 
-	case strings.Contains(enc, "gzip"):
+	// Find supported encoder by accepted list by precedence
-		// TODO: Exception for old MSIE browsers that can't handle non-HTML?
+	for _, name := range encodingPrecedence {
-		// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+		if fn, ok := encoders[name]; ok && matchAcceptEncoding(accepted, name) {
-		return encodingGzip
+			return fn, name
-
+		}
-	case strings.Contains(enc, "deflate"):
-		// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
-		// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
-		// checksum compared to CRC-32 used in "gzip" and thus is faster.
-		//
-		// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
-		// raw DEFLATE data only, without the mentioned zlib wrapper.
-		// Because of this major confusion, most modern browsers try it
-		// both ways, first looking for zlib headers.
-		// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
-		//
-		// The list of browsers having problems is quite big, see:
-		// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
-		// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
-		//
-		// That's why we prefer gzip over deflate. It's just more reliable
-		// and not significantly slower than gzip.
-		return encodingDeflate
-
-		// NOTE: Not implemented, intentionally:
-		// case "compress": // LZW. Deprecated.
-		// case "bzip2":    // Too slow on-the-fly.
-		// case "zopfli":   // Too slow on-the-fly.
-		// case "xz":       // Too slow on-the-fly.
 	}
 
-	return encodingNone
+	// No encoder found to match the accepted encoding
+	return nil, ""
 }
 
-type maybeCompressResponseWriter struct {
+func matchAcceptEncoding(accepted []string, encoding string) bool {
+	for _, v := range accepted {
+		if strings.Index(v, encoding) >= 0 {
+			return true
+		}
+	}
+	return false
+}
+
+type compressResponseWriter struct {
 	http.ResponseWriter
 	w            io.Writer
-	encoding     encoding
+	encoder      EncoderFunc
+	encoding     string
 	contentTypes map[string]struct{}
 	level        int
 	wroteHeader  bool
 }
 
-func (w *maybeCompressResponseWriter) WriteHeader(code int) {
+func (cw *compressResponseWriter) WriteHeader(code int) {
-	if w.wroteHeader {
+	if cw.wroteHeader {
 		return
 	}
-	w.wroteHeader = true
+	cw.wroteHeader = true
-	defer w.ResponseWriter.WriteHeader(code)
+	defer cw.ResponseWriter.WriteHeader(code)
 
 	// Already compressed data?
-	if w.ResponseWriter.Header().Get("Content-Encoding") != "" {
+	if cw.Header().Get("Content-Encoding") != "" {
 		return
 	}
-	// The content-length after compression is unknown
-	w.ResponseWriter.Header().Del("Content-Length")
 
 	// Parse the first part of the Content-Type response header.
 	contentType := ""
-	parts := strings.Split(w.ResponseWriter.Header().Get("Content-Type"), ";")
+	parts := strings.Split(cw.Header().Get("Content-Type"), ";")
 	if len(parts) > 0 {
 		contentType = parts[0]
 	}
 
 	// Is the content type compressable?
-	if _, ok := w.contentTypes[contentType]; !ok {
+	if _, ok := cw.contentTypes[contentType]; !ok {
 		return
 	}
 
-	// Select the compress writer.
+	if cw.encoder != nil && cw.encoding != "" {
-	switch w.encoding {
+		if wr := cw.encoder(cw.ResponseWriter, cw.level); wr != nil {
-	case encodingGzip:
+			cw.w = wr
-		gw, err := gzip.NewWriterLevel(w.ResponseWriter, w.level)
+			cw.Header().Set("Content-Encoding", cw.encoding)
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
-		}
-		w.w = gw
-		w.ResponseWriter.Header().Set("Content-Encoding", "gzip")
 
-	case encodingDeflate:
+			// The content-length after compression is unknown
-		dw, err := flate.NewWriter(w.ResponseWriter, w.level)
+			cw.Header().Del("Content-Length")
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
 		}
-		w.w = dw
-		w.ResponseWriter.Header().Set("Content-Encoding", "deflate")
 	}
 }
 
-func (w *maybeCompressResponseWriter) Write(p []byte) (int, error) {
+func (cw *compressResponseWriter) Write(p []byte) (int, error) {
-	if !w.wroteHeader {
+	if !cw.wroteHeader {
-		w.WriteHeader(http.StatusOK)
+		cw.WriteHeader(http.StatusOK)
 	}
 
-	return w.w.Write(p)
+	return cw.w.Write(p)
 }
 
-func (w *maybeCompressResponseWriter) Flush() {
+func (cw *compressResponseWriter) Flush() {
-	if f, ok := w.w.(http.Flusher); ok {
+	if f, ok := cw.w.(http.Flusher); ok {
 		f.Flush()
 	}
 }
 
-func (w *maybeCompressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	if hj, ok := w.w.(http.Hijacker); ok {
+	if hj, ok := cw.w.(http.Hijacker); ok {
 		return hj.Hijack()
 	}
 	return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer")
 }
 
-func (w *maybeCompressResponseWriter) CloseNotify() <-chan bool {
+func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
-	if cn, ok := w.w.(http.CloseNotifier); ok {
+	if ps, ok := cw.w.(http.Pusher); ok {
-		return cn.CloseNotify()
+		return ps.Push(target, opts)
 	}
-
+	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
-	// If the underlying writer does not implement http.CloseNotifier, return
-	// a channel that never receives a value. The semantics here is that the
-	// client never disconnnects before the request is processed by the
-	// http.Handler, which is close enough to the default behavior (when
-	// CloseNotify() is not even called).
-	return make(chan bool, 1)
 }
 
-func (w *maybeCompressResponseWriter) Close() error {
+func (cw *compressResponseWriter) Close() error {
-	if c, ok := w.w.(io.WriteCloser); ok {
+	if c, ok := cw.w.(io.WriteCloser); ok {
 		return c.Close()
 	}
 	return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer")
 }
+
+func encoderGzip(w http.ResponseWriter, level int) io.Writer {
+	gw, err := gzip.NewWriterLevel(w, level)
+	if err != nil {
+		return nil
+	}
+	return gw
+}
+
+func encoderDeflate(w http.ResponseWriter, level int) io.Writer {
+	dw, err := flate.NewWriter(w, level)
+	if err != nil {
+		return nil
+	}
+	return dw
+}

backend/vendor/github.com/go-chi/chi/middleware/compress18.go

@@ -1,15 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"errors"
-	"net/http"
-)
-
-func (w *maybeCompressResponseWriter) Push(target string, opts *http.PushOptions) error {
-	if ps, ok := w.w.(http.Pusher); ok {
-		return ps.Push(target, opts)
-	}
-	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
-}

backend/vendor/github.com/go-chi/chi/middleware/content_type.go

@@ -26,6 +26,12 @@ func AllowContentType(contentTypes ...string) func(next http.Handler) http.Handl
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
+			if r.ContentLength == 0 {
+				// skip check for empty content body
+				next.ServeHTTP(w, r)
+				return
+			}
+
 			s := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type")))
 			if i := strings.Index(s, ";"); i > -1 {
 				s = s[0:i]

backend/vendor/github.com/go-chi/chi/middleware/logger.go

@@ -16,7 +16,7 @@ var (
 	// DefaultLogger is called by the Logger middleware handler to log each request.
 	// Its made a package-level variable so that it can be reconfigured for custom
 	// logging configurations.
-	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags)})
+	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags), NoColor: false})
 )
 
 // Logger is a middleware that logs the start and end of each request, along
@@ -81,29 +81,32 @@ type LoggerInterface interface {
 
 // DefaultLogFormatter is a simple logger that implements a LogFormatter.
 type DefaultLogFormatter struct {
-	Logger LoggerInterface
+	Logger  LoggerInterface
+	NoColor bool
 }
 
 // NewLogEntry creates a new LogEntry for the request.
 func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
+	useColor := !l.NoColor
 	entry := &defaultLogEntry{
 		DefaultLogFormatter: l,
 		request:             r,
 		buf:                 &bytes.Buffer{},
+		useColor:            useColor,
 	}
 
 	reqID := GetReqID(r.Context())
 	if reqID != "" {
-		cW(entry.buf, nYellow, "[%s] ", reqID)
+		cW(entry.buf, useColor, nYellow, "[%s] ", reqID)
 	}
-	cW(entry.buf, nCyan, "\"")
+	cW(entry.buf, useColor, nCyan, "\"")
-	cW(entry.buf, bMagenta, "%s ", r.Method)
+	cW(entry.buf, useColor, bMagenta, "%s ", r.Method)
 
 	scheme := "http"
 	if r.TLS != nil {
 		scheme = "https"
 	}
-	cW(entry.buf, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
+	cW(entry.buf, useColor, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
 
 	entry.buf.WriteString("from ")
 	entry.buf.WriteString(r.RemoteAddr)
@@ -114,33 +117,34 @@ func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
 
 type defaultLogEntry struct {
 	*DefaultLogFormatter
-	request *http.Request
+	request  *http.Request
-	buf     *bytes.Buffer
+	buf      *bytes.Buffer
+	useColor bool
 }
 
 func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 	switch {
 	case status < 200:
-		cW(l.buf, bBlue, "%03d", status)
+		cW(l.buf, l.useColor, bBlue, "%03d", status)
 	case status < 300:
-		cW(l.buf, bGreen, "%03d", status)
+		cW(l.buf, l.useColor, bGreen, "%03d", status)
 	case status < 400:
-		cW(l.buf, bCyan, "%03d", status)
+		cW(l.buf, l.useColor, bCyan, "%03d", status)
 	case status < 500:
-		cW(l.buf, bYellow, "%03d", status)
+		cW(l.buf, l.useColor, bYellow, "%03d", status)
 	default:
-		cW(l.buf, bRed, "%03d", status)
+		cW(l.buf, l.useColor, bRed, "%03d", status)
 	}
 
-	cW(l.buf, bBlue, " %dB", bytes)
+	cW(l.buf, l.useColor, bBlue, " %dB", bytes)
 
 	l.buf.WriteString(" in ")
 	if elapsed < 500*time.Millisecond {
-		cW(l.buf, nGreen, "%s", elapsed)
+		cW(l.buf, l.useColor, nGreen, "%s", elapsed)
 	} else if elapsed < 5*time.Second {
-		cW(l.buf, nYellow, "%s", elapsed)
+		cW(l.buf, l.useColor, nYellow, "%s", elapsed)
 	} else {
-		cW(l.buf, nRed, "%s", elapsed)
+		cW(l.buf, l.useColor, nRed, "%s", elapsed)
 	}
 
 	l.Logger.Print(l.buf.String())
@@ -148,7 +152,7 @@ func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 
 func (l *defaultLogEntry) Panic(v interface{}, stack []byte) {
 	panicEntry := l.NewLogEntry(l.request).(*defaultLogEntry)
-	cW(panicEntry.buf, bRed, "panic: %+v", v)
+	cW(panicEntry.buf, l.useColor, bRed, "panic: %+v", v)
 	l.Logger.Print(panicEntry.buf.String())
 	l.Logger.Print(string(stack))
 }

backend/vendor/github.com/go-chi/chi/middleware/nocache.go

@@ -14,7 +14,7 @@ var epoch = time.Unix(0, 0).Format(time.RFC1123)
 // Taken from https://github.com/mytrile/nocache
 var noCacheHeaders = map[string]string{
 	"Expires":         epoch,
-	"Cache-Control":   "no-cache, no-store, must-revalidate, private, max-age=0",
+	"Cache-Control":   "no-cache, no-store, no-transform, must-revalidate, private, max-age=0",
 	"Pragma":          "no-cache",
 	"X-Accel-Expires": "0",
 }

backend/vendor/github.com/go-chi/chi/middleware/realip.go

@@ -22,7 +22,7 @@ var xRealIP = http.CanonicalHeaderKey("X-Real-IP")
 // You should only use this middleware if you can trust the headers passed to
 // you (in particular, the two headers this middleware uses), for example
 // because you have placed a reverse proxy like HAProxy or nginx in front of
-// Goji. If your reverse proxies are configured to pass along arbitrary header
+// chi. If your reverse proxies are configured to pass along arbitrary header
 // values from the client, or if you use this middleware without a reverse
 // proxy, malicious clients will be able to make you very sad (or, depending on
 // how you're using RemoteAddr, vulnerable to an attack of some sort).

backend/vendor/github.com/go-chi/chi/middleware/request_id.go

@@ -17,7 +17,7 @@ import (
 // Key to use when setting the request ID.
 type ctxKeyRequestID int
 
-// RequestIDKey is the key that holds th unique request ID in a request context.
+// RequestIDKey is the key that holds the unique request ID in a request context.
 const RequestIDKey ctxKeyRequestID = 0
 
 var prefix string
@@ -62,9 +62,13 @@ func init() {
 // counter.
 func RequestID(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
-		myid := atomic.AddUint64(&reqid, 1)
 		ctx := r.Context()
-		ctx = context.WithValue(ctx, RequestIDKey, fmt.Sprintf("%s-%06d", prefix, myid))
+		requestID := r.Header.Get("X-Request-Id")
+		if requestID == "" {
+			myid := atomic.AddUint64(&reqid, 1)
+			requestID = fmt.Sprintf("%s-%06d", prefix, myid)
+		}
+		ctx = context.WithValue(ctx, RequestIDKey, requestID)
 		next.ServeHTTP(w, r.WithContext(ctx))
 	}
 	return http.HandlerFunc(fn)

backend/vendor/github.com/go-chi/chi/middleware/strip.go

@@ -1,6 +1,7 @@
 package middleware
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/go-chi/chi"
@@ -28,6 +29,9 @@ func StripSlashes(next http.Handler) http.Handler {
 
 // RedirectSlashes is a middleware that will match request paths with a trailing
 // slash and redirect to the same path, less the trailing slash.
+//
+// NOTE: RedirectSlashes middleware is *incompatible* with http.FileServer,
+// see https://github.com/go-chi/chi/issues/343
 func RedirectSlashes(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		var path string
@@ -38,7 +42,11 @@ func RedirectSlashes(next http.Handler) http.Handler {
 			path = r.URL.Path
 		}
 		if len(path) > 1 && path[len(path)-1] == '/' {
-			path = path[:len(path)-1]
+			if r.URL.RawQuery != "" {
+				path = fmt.Sprintf("%s?%s", path[:len(path)-1], r.URL.RawQuery)
+			} else {
+				path = path[:len(path)-1]
+			}
 			http.Redirect(w, r, path, 301)
 			return
 		}

backend/vendor/github.com/go-chi/chi/middleware/terminal.go

@@ -52,12 +52,12 @@ func init() {
 }
 
 // colorWrite
-func cW(w io.Writer, color []byte, s string, args ...interface{}) {
+func cW(w io.Writer, useColor bool, color []byte, s string, args ...interface{}) {
-	if isTTY {
+	if isTTY && useColor {
 		w.Write(color)
 	}
 	fmt.Fprintf(w, s, args...)
-	if isTTY {
+	if isTTY && useColor {
 		w.Write(reset)
 	}
 }

backend/vendor/github.com/go-chi/chi/middleware/timeout.go

@@ -15,7 +15,8 @@ import (
 //
 // ie. a route/handler may look like:
 //
-//  r.Get("/long", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+//  r.Get("/long", func(w http.ResponseWriter, r *http.Request) {
+// 	 ctx := r.Context()
 // 	 processTime := time.Duration(rand.Intn(4)+1) * time.Second
 //
 // 	 select {

backend/vendor/github.com/go-chi/chi/middleware/wrap_writer.go

@@ -10,6 +10,32 @@ import (
 	"net/http"
 )
 
+// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
+// hook into various parts of the response process.
+func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
+	_, fl := w.(http.Flusher)
+
+	bw := basicWriter{ResponseWriter: w}
+
+	if protoMajor == 2 {
+		_, ps := w.(http.Pusher)
+		if fl && ps {
+			return &http2FancyWriter{bw}
+		}
+	} else {
+		_, hj := w.(http.Hijacker)
+		_, rf := w.(io.ReaderFrom)
+		if fl && hj && rf {
+			return &httpFancyWriter{bw}
+		}
+	}
+	if fl {
+		return &flushWriter{bw}
+	}
+
+	return &bw
+}
+
 // WrapResponseWriter is a proxy around an http.ResponseWriter that allows you to hook
 // into various parts of the response process.
 type WrapResponseWriter interface {
@@ -47,6 +73,7 @@ func (b *basicWriter) WriteHeader(code int) {
 		b.ResponseWriter.WriteHeader(code)
 	}
 }
+
 func (b *basicWriter) Write(buf []byte) (int, error) {
 	b.WriteHeader(http.StatusOK)
 	n, err := b.ResponseWriter.Write(buf)
@@ -60,20 +87,25 @@ func (b *basicWriter) Write(buf []byte) (int, error) {
 	b.bytes += n
 	return n, err
 }
+
 func (b *basicWriter) maybeWriteHeader() {
 	if !b.wroteHeader {
 		b.WriteHeader(http.StatusOK)
 	}
 }
+
 func (b *basicWriter) Status() int {
 	return b.code
 }
+
 func (b *basicWriter) BytesWritten() int {
 	return b.bytes
 }
+
 func (b *basicWriter) Tee(w io.Writer) {
 	b.tee = w
 }
+
 func (b *basicWriter) Unwrap() http.ResponseWriter {
 	return b.ResponseWriter
 }
@@ -83,13 +115,15 @@ type flushWriter struct {
 }
 
 func (f *flushWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
 
 var _ http.Flusher = &flushWriter{}
 
-// httpFancyWriter is a HTTP writer that additionally satisfies http.CloseNotifier,
+// httpFancyWriter is a HTTP writer that additionally satisfies
 // http.Flusher, http.Hijacker, and io.ReaderFrom. It exists for the common case
 // of wrapping the http.ResponseWriter that package http gives you, in order to
 // make the proxied object support the full method set of the proxied object.
@@ -97,18 +131,22 @@ type httpFancyWriter struct {
 	basicWriter
 }
 
-func (f *httpFancyWriter) CloseNotify() <-chan bool {
-	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
-	return cn.CloseNotify()
-}
 func (f *httpFancyWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
+
 func (f *httpFancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 	hj := f.basicWriter.ResponseWriter.(http.Hijacker)
 	return hj.Hijack()
 }
+
+func (f *http2FancyWriter) Push(target string, opts *http.PushOptions) error {
+	return f.basicWriter.ResponseWriter.(http.Pusher).Push(target, opts)
+}
+
 func (f *httpFancyWriter) ReadFrom(r io.Reader) (int64, error) {
 	if f.basicWriter.tee != nil {
 		n, err := io.Copy(&f.basicWriter, r)
@@ -122,12 +160,12 @@ func (f *httpFancyWriter) ReadFrom(r io.Reader) (int64, error) {
 	return n, err
 }
 
-var _ http.CloseNotifier = &httpFancyWriter{}
 var _ http.Flusher = &httpFancyWriter{}
 var _ http.Hijacker = &httpFancyWriter{}
+var _ http.Pusher = &http2FancyWriter{}
 var _ io.ReaderFrom = &httpFancyWriter{}
 
-// http2FancyWriter is a HTTP2 writer that additionally satisfies http.CloseNotifier,
+// http2FancyWriter is a HTTP2 writer that additionally satisfies
 // http.Flusher, and io.ReaderFrom. It exists for the common case
 // of wrapping the http.ResponseWriter that package http gives you, in order to
 // make the proxied object support the full method set of the proxied object.
@@ -135,14 +173,11 @@ type http2FancyWriter struct {
 	basicWriter
 }
 
-func (f *http2FancyWriter) CloseNotify() <-chan bool {
-	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
-	return cn.CloseNotify()
-}
 func (f *http2FancyWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
 
-var _ http.CloseNotifier = &http2FancyWriter{}
 var _ http.Flusher = &http2FancyWriter{}

backend/vendor/github.com/go-chi/chi/middleware/wrap_writer17.go

@@ -1,34 +0,0 @@
-// +build go1.7,!go1.8
-
-package middleware
-
-import (
-	"io"
-	"net/http"
-)
-
-// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
-// hook into various parts of the response process.
-func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
-	_, cn := w.(http.CloseNotifier)
-	_, fl := w.(http.Flusher)
-
-	bw := basicWriter{ResponseWriter: w}
-
-	if protoMajor == 2 {
-		if cn && fl {
-			return &http2FancyWriter{bw}
-		}
-	} else {
-		_, hj := w.(http.Hijacker)
-		_, rf := w.(io.ReaderFrom)
-		if cn && fl && hj && rf {
-			return &httpFancyWriter{bw}
-		}
-	}
-	if fl {
-		return &flushWriter{bw}
-	}
-
-	return &bw
-}

backend/vendor/github.com/go-chi/chi/middleware/wrap_writer18.go

@@ -1,41 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"io"
-	"net/http"
-)
-
-// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
-// hook into various parts of the response process.
-func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
-	_, cn := w.(http.CloseNotifier)
-	_, fl := w.(http.Flusher)
-
-	bw := basicWriter{ResponseWriter: w}
-
-	if protoMajor == 2 {
-		_, ps := w.(http.Pusher)
-		if cn && fl && ps {
-			return &http2FancyWriter{bw}
-		}
-	} else {
-		_, hj := w.(http.Hijacker)
-		_, rf := w.(io.ReaderFrom)
-		if cn && fl && hj && rf {
-			return &httpFancyWriter{bw}
-		}
-	}
-	if fl {
-		return &flushWriter{bw}
-	}
-
-	return &bw
-}
-
-func (f *http2FancyWriter) Push(target string, opts *http.PushOptions) error {
-	return f.basicWriter.ResponseWriter.(http.Pusher).Push(target, opts)
-}
-
-var _ http.Pusher = &http2FancyWriter{}

backend/vendor/github.com/go-chi/chi/mux.go

@@ -60,7 +60,8 @@ func NewMux() *Mux {
 func (mx *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Ensure the mux has some routes defined on the mux
 	if mx.handler == nil {
-		panic("chi: attempting to route to a mux with no handlers.")
+		mx.NotFoundHandler().ServeHTTP(w, r)
+		return
 	}
 
 	// Check if a routing context already exists from a parent router.

backend/vendor/github.com/go-chi/chi/tree.go

@@ -33,15 +33,15 @@ var mALL = mCONNECT | mDELETE | mGET | mHEAD |
 	mOPTIONS | mPATCH | mPOST | mPUT | mTRACE
 
 var methodMap = map[string]methodTyp{
-	"CONNECT": mCONNECT,
+	http.MethodConnect: mCONNECT,
-	"DELETE":  mDELETE,
+	http.MethodDelete:  mDELETE,
-	"GET":     mGET,
+	http.MethodGet:     mGET,
-	"HEAD":    mHEAD,
+	http.MethodHead:    mHEAD,
-	"OPTIONS": mOPTIONS,
+	http.MethodOptions: mOPTIONS,
-	"PATCH":   mPATCH,
+	http.MethodPatch:   mPATCH,
-	"POST":    mPOST,
+	http.MethodPost:    mPOST,
-	"PUT":     mPUT,
+	http.MethodPut:     mPUT,
-	"TRACE":   mTRACE,
+	http.MethodTrace:   mTRACE,
 }
 
 // RegisterMethod adds support for custom HTTP method handlers, available
@@ -706,7 +706,9 @@ func patNextSegment(pattern string) (nodeTyp, string, string, byte, int, int) {
 	}
 
 	// Wildcard pattern as finale
-	// TODO: should we panic if there is stuff after the * ???
+	if ws < len(pattern)-1 {
+		panic("chi: wildcard '*' must be the last value in a route. trim trailing text or use a '{param}' instead")
+	}
 	return ntCatchAll, "*", "", 0, ws, len(pattern)
 }
 

backend/vendor/github.com/go-pkgz/auth/.travis.yml

@@ -10,7 +10,7 @@ install: true
 
 before_install:
   - export TZ=America/Chicago
-  - curl -L https://git.io/vp6lP | sh 
+  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2
   - go get github.com/mattn/goveralls
   - export MONGO_TEST=mongodb://127.0.0.1:27017
   - export PATH=$(pwd)/bin:$PATH
@@ -19,5 +19,5 @@ script:
   - GO111MODULE=on go get ./...
   - GO111MODULE=on go mod vendor
   - GO111MODULE=on go test  -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
-  - ./bin/gometalinter --deadline=120s --exclude=test --exclude=mock --exclude=vendor --exclude=_example --disable-all --enable=errcheck --enable=vet --enable=vetshadow --enable=megacheck --enable=ineffassign --enable=varcheck --enable=unconvert --enable=deadcode --enable=interfacer --enable=gotype ./... || travis_terminate 1;
+  - golangci-lint run --tests=false  || travis_terminate 1;
   - $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci

backend/vendor/github.com/go-pkgz/auth/README.md

@@ -10,7 +10,8 @@ This library provides "social login" with Github, Google, Facebook and Yandex as
 - Minimal scopes with user name, id and picture (avatar) only
 - Direct authentication with user's provided credential checker
 - Integrated avatar proxy with FS, boltdb and gridfs storages
-- Support of user-defined storages for avatars
+- Support of user-defined storage for avatars
+- Identicon for default avatars  
 - Black list with user-defined validator
 - Multiple aud (audience) supported
 - Secure key with customizable `SecretReader`
@@ -21,7 +22,7 @@ This library provides "social login" with Github, Google, Facebook and Yandex as
 
 ## Install
 
-`go install github.com/go-pkgz/auth`
+`go get -u github.com/go-pkgz/auth`
 
 ## Usage
 
@@ -144,6 +145,8 @@ Such provider acts like any other, i.e. will be registered as `/auth/local/login
 
 The API for this provider - `GET /auth/<name>/login?user=<user>&passwd=<password>&aud=<site_id>&session=[1|0]`
 
+_note: password parameter doesn't have to be naked/real password and can be any kind of password hash prepared by caller._ 
+  
 ### Customization
 
 There are several ways to adjust functionality of the library:

backend/vendor/github.com/go-pkgz/auth/auth.go

@@ -1,3 +1,4 @@
+// Package auth provides "social login" with Github, Google, Facebook and Yandex as well as custom auth providers.
 package auth
 
 import (
@@ -146,6 +147,11 @@ func (s *Service) Handlers() (authHandler http.Handler, avatarHandler http.Handl
 
 		// allow logout without specifying provider
 		if elems[len(elems)-1] == "logout" {
+			if len(s.providers) == 0 {
+				w.WriteHeader(http.StatusBadRequest)
+				rest.RenderJSON(w, r, rest.JSON{"error": "provides not defined"})
+				return
+			}
 			s.providers[0].Handler(w, r)
 			return
 		}
@@ -202,7 +208,7 @@ func (s *Service) AddProvider(name string, cid string, csecret string) {
 	case "facebook":
 		s.providers = append(s.providers, provider.NewService(provider.NewFacebook(p)))
 	case "yandex":
-		s.providers = append(s.providers, provider.NewService(provider.NewFacebook(p)))
+		s.providers = append(s.providers, provider.NewService(provider.NewYandex(p)))
 	case "dev":
 		s.providers = append(s.providers, provider.NewService(provider.NewDev(p)))
 	default:
@@ -221,6 +227,7 @@ func (s *Service) AddDirectProvider(name string, credChecker provider.CredChecke
 		Issuer:       s.issuer,
 		TokenService: s.jwtService,
 		CredChecker:  credChecker,
+		AvatarSaver:  s.avatarProxy,
 	}
 	s.providers = append(s.providers, provider.NewService(dh))
 	s.authMiddleware.Providers = s.providers

backend/vendor/github.com/go-pkgz/auth/avatar/avatar.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/go-pkgz/rest"
+	"github.com/nullrocks/identicon"
 	"github.com/pkg/errors"
 	"golang.org/x/image/draw"
 
@@ -33,9 +34,19 @@ type Proxy struct {
 // Put stores retrieved avatar to avatar.Store. Gets image from user info. Returns proxied url
 func (p *Proxy) Put(u token.User) (avatarURL string, err error) {
 
-	// no picture for user, try default avatar
+	// no picture for user, try to generate identicon avatar
 	if u.Picture == "" {
-		return "", errors.Errorf("no picture for %s", u.ID)
+		b, err := GenerateAvatar(u.ID)
+		if err != nil {
+			return "", errors.Errorf("no picture for %s", u.ID)
+		}
+		avatarID, err := p.Store.Put(u.ID, p.resize(bytes.NewBuffer(b), p.ResizeLimit)) // put returns avatar base name, like 123456.image
+		if err != nil {
+			return "", err
+		}
+
+		p.Logf("[DEBUG] saved identicon avatar to %s, user %q", avatarID, u.Name)
+		return p.URL + p.RoutePath + "/" + avatarID, nil
 	}
 
 	// load avatar from remote location
@@ -152,6 +163,25 @@ func (p *Proxy) resize(reader io.Reader, limit int) io.Reader {
 	}
 	return &out
 }
+
+// GenerateAvatar for give user with identicon
+func GenerateAvatar(user string) ([]byte, error) {
+
+	iconGen, err := identicon.New("pkgz/auth", 5, 5)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create identicon service")
+	}
+
+	ii, err := iconGen.Draw(user) // generate an IdentIcon
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to draw avatar for %s", user)
+	}
+
+	buf := &bytes.Buffer{}
+	err = ii.Png(300, buf)
+	return buf.Bytes(), err
+}
+
 func retry(retries int, delay time.Duration, fn func() error) (err error) {
 	for i := 0; i < retries; i++ {
 		if err = fn(); err == nil {